In [1]:
%matplotlib inline
import matplotlib.pyplot as plt
import os
import numpy as np
import pandas as pd
import statsmodels.api as sm
from statsmodels.formula.api import ols
pd.set_option('display.max_columns', 8)
In [2]:
from sklearn import datasets
dataset=datasets.load_boston()
X=dataset.data
y=dataset.target
df=pd.DataFrame(dataset.data,columns=dataset.feature_names)
df.loc[:,'target']=dataset.target
df.head()
Out[2]:
In [3]:
df.corr()
Out[3]:
In [4]:
from sklearn.preprocessing import StandardScaler
scaler=StandardScaler()
X=scaler.fit_transform(X)
1. OLS¶
- sklearn
In [5]:
from sklearn.linear_model import LinearRegression
regr=LinearRegression().fit(X,y)
regr.score(X,y)
Out[5]:
In [6]:
regr.predict(X)
regr.coef_#Coefficients
Out[6]:
In [7]:
df1=df.iloc[:,:-1]
In [8]:
def vif(df, col_i):
cols = list(df.columns)
cols.remove(col_i)
cols_noti = cols
formula = col_i + '~' + '+'.join(cols_noti)
r2 = ols(formula, df).fit().rsquared
return 1. / (1. - r2)
- statsmodels
In [9]:
lm = ols('target ~ '+'+'.join(dataset.feature_names),data=df).fit()
lm.summary()
Out[9]:
2. 岭回归(ridge)¶
- sklearn
In [10]:
from sklearn.linear_model import Ridge
ridge = Ridge()
alphas = np.logspace(-2, 5, 1000, base=10)
coefs = []
scores=[]
for alpha in alphas:
ridge.set_params(alpha=alpha)
ridge.fit(X, y)
coefs.append(ridge.coef_)
scores.append(ridge.score(X,y))
In [11]:
ax = plt.gca()
ax.plot(alphas, coefs)
ax.set_xscale('log')
plt.xlabel('alpha')
plt.ylabel('weights')
plt.title('Ridge coefficients as a function of the regularization')
plt.axis('tight')
plt.show()
plt.figure()
plt.plot(alphas,scores)
Out[11]:
In [12]:
ridge.set_params(alpha=4.67)
ridge.fit(X, y)
Out[12]:
In [13]:
ridge.predict(X)
ridge.coef_
ridge.score(X, y)
Out[13]:
- statsmodels
In [14]:
lmr = ols('target ~ '+'+'.join(dataset.feature_names),data=df).fit_regularized(alpha=1, L1_wt=0)
lmr.summary()
# L1_wt参数为0则使用岭回归，为1使用lasso
In [15]:
lmr.params
Out[15]:
In [16]:
from sklearn.preprocessing import StandardScaler
scaler=StandardScaler()
X = scaler.fit_transform(df.iloc[:,:-1])
y = df.loc[:,'target']
In [17]:
from sklearn.linear_model import RidgeCV
alphas = np.logspace(-2, 5, 1000, base=10)
# Search the min MSE by CV
rcv = RidgeCV(alphas=alphas, store_cv_values=True)
rcv.fit(X, y)
Out[17]:
In [18]:
rcv.predict(X)
print('The best alpha is {}'.format(rcv.alpha_))
print('The r-square is {}'.format(rcv.score(X, y)))
# Default score is rsquared
rcv.coef_
Out[18]:
In [19]:
cv_values = rcv.cv_values_
n_fold, n_alphas = cv_values.shape
cv_mean = cv_values.mean(axis=0)
cv_std = cv_values.std(axis=0)
ub = cv_mean + cv_std / np.sqrt(n_fold)
lb = cv_mean - cv_std / np.sqrt(n_fold)
plt.semilogx(alphas, cv_mean, label='mean_score')
plt.fill_between(alphas, lb, ub, alpha=0.2)
plt.xlabel(r"$\alpha$")
plt.ylabel("mean squared errors")
plt.legend(loc="best")
plt.show()
#这个图，最低点最好
4. LASSO¶
- statsmodels
In [20]:
lmr1 = ols('target ~ '+'+'.join(dataset.feature_names),data=df).fit_regularized(alpha=1, L1_wt=1)
lmr1.summary()
- sklearn.lnear_model.LassoCV
In [21]:
from sklearn.linear_model import LassoCV
lasso_alphas = np.logspace(-3, 2, 100, base=10)
lcv = LassoCV(alphas=lasso_alphas, cv=10) # Search the min MSE by CV
lcv.fit(X, y)
print('The best alpha is {}'.format(lcv.alpha_))
print('The r-square is {}'.format(lcv.score(X, y)))
# Default score is rsquared
- sklearn.lnear_model.Lasso
In [22]:
from sklearn.linear_model import Lasso
lasso = Lasso()
lasso_coefs = []
for alpha in lasso_alphas:
lasso.set_params(alpha=alpha)
lasso.fit(X, y)
lasso_coefs.append(lasso.coef_)
In [23]:
ax = plt.gca()
ax.plot(lasso_alphas, lasso_coefs)
ax.set_xscale('log')
plt.xlabel('alpha')
plt.ylabel('weights')
plt.title('Lasso coefficients as a function of the regularization')
plt.axis('tight')
plt.show()
In [3]:
from sklearn.linear_model import ElasticNet#还有ElasticNetCV
regr=ElasticNet(alpha=1.0, l1_ratio=0.5, fit_intercept=True, normalize=False, precompute=False, max_iter=1000, copy_X=True, tol=0.0001,
warm_start=False, positive=False, random_state=None, selection='cyclic')
# alpha:alpha
# l1_ratio:rho值
# fit_intercept:是否拟合intercept