find suitable algorithm
classification
def create_baseline_classifiers(seed=8):
"""Create a list of baseline classifiers.
Parameters
----------
seed: (optional) An integer to set seed for reproducibility
Returns
-------
A list containing tuple of name, model object for each of these algortihms:
DummyClassifier, LogisticRegression, SGDClassifier, ExtraTreesClassifier,
GradientBoostingClassifier, RandomForestClassifier, MultinomialNB, SVC,
XGBClassifier.
"""
models = []
models.append(('dum', DummyClassifier(random_state=seed, strategy='most_frequent')))
models.append(('log', LogisticRegression(random_state=seed)))
models.append(('sgd', SGDClassifier(random_state=seed)))
models.append(('etc', ExtraTreesClassifier(random_state=seed)))
models.append(('gbm', GradientBoostingClassifier(random_state=seed)))
models.append(('rfc', RandomForestClassifier(random_state=seed)))
models.append(('mnb', MultinomialNB()))
models.append(('svc', SVC(random_state=seed, probability=True)))
models.append(('xgb', XGBClassifier(seed=seed)))
return models
def assess_models(X, y, models, cv=5, metrics=['roc_auc', 'f1']):
"""Provide summary of cross validation results for models.
Parameters
----------
X: A pandas DataFrame containing feature matrix
y: A pandas Series containing target vector
models: A list of models to train
cv: (optional) An integer to set number of folds in cross-validation
metrics: (optional) A list of scoring metrics or a string for a metric
Returns
-------
A pandas DataFrame containing summary of baseline models' performance.
"""
summary = pd.DataFrame()
for name, model in models:
result = pd.DataFrame(cross_validate(model, X, y, cv=cv, scoring=metrics))
mean = result.mean().rename('{}_mean'.format)
std = result.std().rename('{}_std'.format)
summary[name] = pd.concat([mean, std], axis=0)
return summary.sort_index()
def extract_metric(summary, metric):
"""Provide summary of baseline models' performance for a metric.
Parameters
----------
summary: A pandas DataFrame containing the summary of baseline models
metric: A string specifying the name of the metric to extract info
Returns
-------
A pandas DataFrame containing mean, standard deviation, lower and upper
bound of the baseline models' performance in cross validation according to
the metric specified.
"""
output = summary[summary.index.str.contains(metric)].T
output.columns = output.columns.str.replace(f'test_{metric}_', '')
output.sort_values(by='mean', ascending=False, inplace=True)
output['lower'] = output['mean'] - 2*output['std']
output['upper'] = output['mean'] + 2*output['std']
return output
models = create_baseline_classifiers()
summary = assess_models(X_train_transformed, y_train, models)
extract_metric(summary, 'roc_auc')
regression
from sklearn.dummy import DummyRegressor
from sklearn.linear_model import LinearRegression, SGDRegressor
from sklearn.ensemble import ExtraTreesRegressor, GradientBoostingRegressor, RandomForestRegressor
from sklearn.svm import SVR
from xgboost.sklearn import XGBRegressor
def create_baseline_regressors(seed=8):
"""Create a list of of baseline regressors.
Parameters
----------
seed: (optional) An integer to set seed for reproducibility
Returns
-------
A list containing tuple of name, model object for each of these algortihms:
DummyRegressor, LinearRegression, SGDRegressor, ExtraTreesRegressor,
GradientBoostingRegressor, RandomForestRegressor, SVR, XGBRegressor.
"""
models = []
models.append(('dum', DummyRegressor(strategy='mean')))
models.append(('ols', LinearRegression()))
models.append(('sgd', SGDRegressor(random_state=seed)))
models.append(('etr', ExtraTreesRegressor(random_state=seed)))
models.append(('gbm', GradientBoostingRegressor(random_state=seed)))
models.append(('rfr', RandomForestRegressor(random_state=seed)))
models.append(('svc', SVR()))
models.append(('xgb', XGBRegressor(seed=seed)))
return models
sklearn pipeline
pipeline can be used to transform features for a machine learning model.
from sklearn.pipeline import Pipeline
from sklearn.impute import SimpleImputer
from sklearn.preprocessing import MinMaxScaler, OneHotEncoder
#num_transformer fills the missing values with the mean value of a column (SimpleImputer) and scales the values between 0 and 1 (MinMaxScaler)
num_transformer = Pipeline(
steps=
[
('imputer', SimpleImputer(strategy='mean')),
('scaler', MinMaxScaler())
]
)
#cat_transformer fills the missing values with the most frequent value of a column and encode the categories using the OneHotEncoder
cat_transformer = Pipeline(
steps=
[
('imputer', SimpleImputer(strategy='most_frequent')),
('encode', OneHotEncoder(drop='first'))
]
)
#create transformer object
num_ft = df.iloc[:,:-1]\
.select_dtypes(include=['int64', 'float64']).columns
cat_ft = df.iloc[:,:-1]\
.select_dtypes(include=['object']).columns
from sklearn.compose import ColumnTransformer
proprocess = ColumnTransformer(
transformers=[
('numeric', num_transformer, num_ft),
('categorical', cat_transformer, cat_ft)
]
)
#combine tranformer object with machine learning model
from sklearn.linear_model import LogisticRegression
clf = Pipeline(
[
('preprocess', preprocess),
('model', LogisticRegression())
]
)
# start training
from sklearn.model_selection import train_test_split
X = df.drop('target', axis=1)
y = df['target']
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2)
clf.fit(X_train, y_train)
Decision Threshold
using scikit-lego and yellowbrick you can tune decision threshold to have a good balance between precision and recall
import pandas as pd
import numpy as np
from sklego.datasets import load_hearts
from sklearn.model_selection import train_test_split
data = load_hearts(as_frame=True)
X, y = data.drop(columns='target'), data['target']
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2)
from sklearn.pipeline import make_union, make_pipeline
from sklearn.preprocessing import OneHotEncoder, StandardScaler
from sklearn.impute import SimpleImputer
#sklego provides a PandasTypeSelector, so that numerical and categrical pipeline are combined
from sklego.preprocessing import PandasTypeSelector
cat_features_preprocessing = make_pipeline(PandasTypeSelector(exclude='number'),
SimpleImputer(strategy='constant', fill_value='unknown'),
OneHotEncoder(categories=[['normal', 'sth', 'fixed']], handle_unknown='ignore'))
num_features_preprocessing = make_pipeline(PandasTypeSelector(include='number'),
SimpleImputer(strategy='median'),
StandardScaler())
preprocessor = make_union(cat_features_preprocessing, num_features_preprocessing)
#train model
from sklearn.linear_model import LogisticRegression
from sklearn.model_selection import GridSearchCV
model = LogisticRegression()
pipe = make_pipeline(preprocessor, model)
param_grid = {'logisticregression__C': np.logspace(-2, 1, 10)}
#elected the best model using average_precision rather than f1 score will produce more flexibility in selecting best model
grid = GridSearchCV(pipe, param_grid, cv=5, scoring='average_precision',
n_jobs=-1, verbose=1, return_train_score=True)
grid.fit(X_train, y_train)
print(grid.best_score_)
best_model = grid.best_estimator_
# show confusion metrix
from yellowbrick.classifier import ConfusionMatrix
cm = ConfusionMatrix(best_model, cmap='Blues')
cm.score(X_train, y_train)
cm.show();
#show threadhold change on precision and recall (f1 is the harmonic mean of the two)
from yellowbrick.classifier import DiscriminationThreshold
visualizer = DiscriminationThreshold(best_model, quantiles=np.array([0.25, 0.5, 0.75]))
visualizer.fit(X_train, y_train)
visualizer.show();
# use argmax and Threadhold to tune the model
from sklego.meta import Thresholder
#visualizer.argmax is equal to f1
best_threshold = visualizer.thresholds_[visualizer.cv_scores_[visualizer.argmax].argmax()]
#get final pipeline using best threadhold
pipe = make_pipeline(*best_model[:-1], Thresholder(best_model[1], best_threshold))
pipe.fit(X_train, y_train);
