SageMaker: Analysis and model training on the Iris dataset¶
Download and read the dataset¶
In [1]:
import urllib
import pandas as pd
import numpy as np
import seaborn as sns
import matplotlib.pyplot as plt
In [2]:
# For attributes and the class, see: https://archive.ics.uci.edu/ml/machine-learning-databases/iris/iris.names
# For the data, see: https://archive.ics.uci.edu/ml/machine-learning-databases/iris/iris.data
download_url = "https://archive.ics.uci.edu/ml/machine-learning-databases/iris/iris.data"
file_name = "iris.data"
urllib.request.urlretrieve (download_url, file_name)
Out[2]:
In [3]:
# Read the data into Panda dataframe
df = pd.read_csv('./{}'.format(file_name), names=['sepal_length', 'sepal_width', 'petal_length', 'petal_width', 'species'])
Inspect the data and perform some simple analysis¶
In [4]:
# First few observations
df.head()
Out[4]:
In [5]:
# Number of observations
df.count()
Out[5]:
In [6]:
# Number of classes / species
df['species'].unique()
Out[6]:
In [7]:
# Summary of attributes, data types
df.info()
In [8]:
# Some statistics - useful or not?
df.describe()
Out[8]:
In [9]:
# Possibly more useful - mean values grouped by the species
df.groupby('species').mean()
Out[9]:
In [10]:
# Distribution of values for each attribute
plt.figure(1 , figsize = (15 , 6))
n = 0
for x in ['sepal_length' , 'sepal_width' , 'petal_length', 'petal_width']:
n += 1
plt.subplot(1 , 4 , n)
#plt.subplots_adjust(hspace =0.5 , wspace = 0.5)
sns.distplot(df[x] , bins = 20)
plt.title('Distplot of {}'.format(x))
plt.show()
Look for relationships and correlation¶
In [11]:
# Correlation heatmap - petal length and width look to be most associated
plt.figure(figsize=(7,4))
sns.heatmap(df.corr(),annot=True)
plt.show()
In [12]:
# An alternative correlation view - shows the strong association between petal length and width
g = sns.pairplot(df, hue='species', markers='+')
plt.show()
Classify using the scikit-learn K-Nearest Neighbor algorithm¶
Prepare the training and test data¶
In [13]:
# Create training and test dataframes based on a random 70/30 split
train_data, test_data = np.split(df.sample(frac=1, random_state=np.random.RandomState()), [int(0.7 * len(df))])
Create a knn model using the training data¶
In [14]:
from sklearn.neighbors import KNeighborsClassifier
# Declare knn classifer; classify based on most common classification of 3 nearest neighbours
knn = KNeighborsClassifier(n_neighbors=3)
# Train knn model
knn.fit(train_data[['sepal_length', 'sepal_width', 'petal_length', 'petal_width']], train_data["species"])
Out[14]:
Predict species using the model and the test data¶
In [15]:
# Predict using the model and the test data
preds_array = knn.predict(test_data[['sepal_length', 'sepal_width', 'petal_length', 'petal_width']])
# Convert the array to a dataframe with a single column called pred
preds_df = pd.DataFrame(preds_array, columns=['prediction'])
# Add the pred dataframe to the test dataframe by simply placing side by side
combined_df = test_data.reset_index(drop=True).join(preds_df)
Review the predictions¶
In [16]:
# Predictions look good compared to the known class
combined_df.head(20)
Out[16]:
In [17]:
# Prediction of species is largely correct compared to the observed species
pd.crosstab(combined_df['species'], combined_df['prediction'], rownames=['Actual species'], colnames=['Predicted species'])
Out[17]:
In [18]:
# Visually, the predictions show what we would expect when compared with the training data pairplot above
sns.pairplot(combined_df, hue='prediction', markers='+')
plt.show()
Classify using the SageMaker k-nearest neighbour algorithm¶
Prepare the training and test data¶
In [19]:
# Reload CSV data into a Panda dataframe
df = pd.read_csv('./{}'.format(file_name), names=['sepal_length', 'sepal_width', 'petal_length', 'petal_width', 'species'])
# Remap species values to integers
df['species'] = df['species'].replace({'Iris-setosa': 0, 'Iris-versicolor': 1, 'Iris-virginica': 2})
# Move species column to first position
df = pd.concat([df['species'], df.drop(['species'], axis=1)], axis=1)
# Create training and test dataframes based on a random 70/30 split
train_data, test_data = np.split(df.sample(frac=1, random_state=np.random.RandomState()), [int(0.7 * len(df))])
Create a knn model using the training data¶
In [20]:
import boto3
from datetime import datetime
import sagemaker
from sagemaker import get_execution_role
from sagemaker.predictor import csv_serializer, json_deserializer
from sagemaker.amazon.amazon_estimator import get_image_uri
# S3 config
bucket = 'iris-2020'
train_fname = 'iris-train.csv'
test_fname = 'iris-test.csv'
output_path = 's3://{}/output'.format(bucket)
# Save training and test data to local notebook instance (without indexes and headers)
train_data.to_csv(train_fname, index=False, header=False)
test_data.to_csv(test_fname, index=False, header=False)
# Save training and test data to S3
boto3.Session().resource('s3').Bucket(bucket).Object("{}/{}".format('train', train_fname)).upload_file(train_fname)
boto3.Session().resource('s3').Bucket(bucket).Object("{}/{}".format('test', test_fname)).upload_file(test_fname)
# Training config
job_name = 'iris-job-{}'.format(datetime.now().strftime("%Y%m%d%H%M%S"))
# Declare knn estimator
knn = sagemaker.estimator.Estimator(get_image_uri(boto3.Session().region_name, "knn"),
get_execution_role(),
train_instance_count=1,
train_instance_type='ml.m4.xlarge',
output_path=output_path,
sagemaker_session=sagemaker.Session())
# Set mandatory hyperparameters; classify based on most common classification of 3 nearest neighbours
knn.set_hyperparameters(predictor_type='classifier',
feature_dim=4,
k=3,
sample_size=len(train_data))
# Define the data type and paths to the training and test data
content_type = "text/csv"
train_input = sagemaker.session.s3_input(s3_data="s3://{}/{}/".format(bucket, 'train'), content_type=content_type)
test_input = sagemaker.session.s3_input(s3_data="s3://{}/{}/".format(bucket, 'test'), content_type=content_type)
# Train the knn model with just training data
knn.fit({'train': train_input}, job_name=job_name)
# Train the knn model with training and validation data
# knn.fit({'train': train_input, 'test': test_input}, job_name=job_name)
Deploy the model on a SageMaker endpoint¶
In [21]:
# Deploy the model to a Sagemaker endpoint
knn_predictor = knn.deploy(initial_instance_count=1,instance_type='ml.m4.xlarge')
knn_predictor.serializer = csv_serializer
knn_predictor.deserializer = json_deserializer
Predict species using the model and our test data¶
In [22]:
# Convert the test dataframe into an array and drop the species column on the fly
test_data_array = test_data.drop(['species'], axis=1).values
# Predict using the model and the test data
preds = knn_predictor.predict(test_data_array)
# Predict using the model and one set of flower values
# preds = knn_predictor.predict([4.8, 3.0, 1.4, 0.1])
# Convert JSON predictions to an array
preds_array = np.array([preds['predictions'][i]['predicted_label'] for i in range(len(preds['predictions']))])
# Convert the array to a dataframe with a single column called pred
preds_df = pd.DataFrame(preds_array, columns=['prediction'])
# Add the pred dataframe to the test dataframe by simply placing side by side
combined_df = test_data.reset_index(drop=True).join(preds_df)
# Remap species integers back to original values
combined_df['species'] = combined_df['species'].replace({0: 'Iris-setosa', 1: 'Iris-versicolor', 2: 'Iris-virginica'})
combined_df['prediction'] = combined_df['prediction'].replace({0: 'Iris-setosa', 1: 'Iris-versicolor', 2: 'Iris-virginica'})
Review the predictions¶
In [23]:
# As with the previous model above, predictions look good compared to the known class
combined_df.head(20)
Out[23]:
In [24]:
# Prediction of species is largely correct compared to the observed species
pd.crosstab(combined_df['species'], combined_df['prediction'], rownames=['Actual species'], colnames=['Predicted species'])
Out[24]:
In [25]:
# Visually, the predictions show what we would expect when compared with the training data pairplot higher up above
sns.pairplot(combined_df, hue='prediction', markers='+')
plt.show()
Delete the endpoint (to avoid running up a big bill)¶
In [26]:
sagemaker.Session().delete_endpoint(knn_predictor.endpoint)