机器学习基石 作业4 带Regularizer和Cross Validation的线性回归分类器

#!/usr/bin/env python
# -*- coding: utf-8 -*-
"""
__title__ = 'main.py'
__author__ = 'w1d2s'
__mtime__ = '2015/10/30'
"""
from numpy import *
from RidgeReg import *
from Validation import *
import sys
import string

def Data_Pretreatment(path):
    rawData = open(path).readlines()
    #print rawData
    dataNum = len(rawData)
    dataDim = len(rawData[0].strip().split(' ')) - 1
    dataIdx = 0
    X = zeros([dataNum, dataDim])
    Y = zeros(dataNum)
    print(dataNum, dataDim)
    for line in rawData:
        tempList = line.strip().split(' ')
        Y[dataIdx] = string.atoi(tempList[dataDim])
        X[dataIdx, :] = tempList[0: dataDim]
        dataIdx += 1
    return (X, Y)



if __name__ == '__main__':
    Xtrain, Ytrain = Data_Pretreatment('train.dat')
    Xtest, Ytest = Data_Pretreatment('test.dat')
    #(Wt, p) = Validate(Xtrain, Ytrain, 120, False)
    (Wt, p) = Cross_Validate(Xtrain, Ytrain, 5)


    rate = 10 ** p
    W = Ridge_Regression(Xtrain, Ytrain, rate)
    Ein = Err_Counter(Xtrain, Ytrain, W)
    Eout = Err_Counter(Xtest, Ytest, W)
    print '** Ein : ' + str(float(Ein)/200)
    print '** Eout : ' + str(float(Eout)/1000)

#!/usr/bin/env python
# -*- coding: utf-8 -*-
"""
__title__ = 'RidgeReg.py'
__author__ = 'w1d2s'
__mtime__ = '2015/10/30'
"""
from numpy import *
from scipy import linalg
import random

def Err_Counter(X, Y, W):
    (dataSize, dataDim) = X.shape
    Z = ones([dataSize, dataDim + 1])
    Z[:, 1: dataDim + 1] = X
    ErrCnt = 0
    for i in range(0, dataSize):
        if Y[i] * dot(Z[i, :], W) <= 0:
            ErrCnt = ErrCnt + 1
    return ErrCnt

def Ridge_Regression(X, Y, rate):
    (dataSize, dataDim) = X.shape
    Z = ones([dataSize, dataDim + 1])
    Z[:, 1: dataDim + 1] = X
    Zt = transpose(Z)
    ZtZ = dot(Zt, Z)
    I = identity(len(ZtZ))
    P = linalg.inv(ZtZ + rate * I)
    W = dot(dot(P, Zt), Y)
    return W

#!/usr/bin/env python
# -*- coding: utf-8 -*-
"""
__title__ = 'Validation.py'
__author__ = 'w1d2s'
__mtime__ = '2015/10/30'
"""
from numpy import *
from RidgeReg import *

def Data_Spliter(X, Y, Num4Train):
    Xtrain = X[0: Num4Train, :]
    Ytrain = Y[0: Num4Train]
    Xval = X[Num4Train: , :]
    Yval = Y[Num4Train: ]
    return [Xtrain, Ytrain, Xval, Yval]

def Validate(X, Y, Num4Train, IsEt):
    [Xt, Yt, Xv, Yv] = Data_Spliter(X, Y, Num4Train)
    minEt = 120
    minEv = 80
    Wt = zeros([1, Xt.ndim + 1])
    p = 0
    for pow in range(-10, 3):
        rate = 10 ** pow
        W = Ridge_Regression(Xt, Yt, rate)
        Et = Err_Counter(Xt, Yt, W)
        Ev = Err_Counter(Xv, Yv, W)
        if IsEt == True:
            if Et <= minEt:
                [Wt, minEt, p] = [W, Et, pow]
            print '== Et : ' + str(float(Et)/120)
            print '== log lambda : ' + str(pow)
        else:
            if Ev <= minEv:
                [Wt, minEv, p] = [W, Ev, pow]
            print '== Ev : ' + str(float(Ev)/80)
            print '== log lambda : ' + str(pow)
    Et = Err_Counter(Xt, Yt, Wt)
    Ev = Err_Counter(Xv, Yv, Wt)
    print 'log lambda : ' + str(p)
    print 'Et : ' + str(float(Et)/120)
    print 'Ev: ' + str(float(Ev)/80)
    return (Wt, p)

def Data_Spliter2(X, Y, folds):
    dataSize = len(Y)
    inc = dataSize / folds
    Xlist = []
    Ylist = []
    for idx in range(0, dataSize, inc):
        Xtemp = X[idx: idx + inc, :]
        Ytemp = Y[idx: idx + inc]
        Xlist.append(Xtemp)
        Ylist.append(Ytemp)
    return (Xlist, Ylist)

def Cross_Validate(X, Y, folds):
    (Xlist, Ylist) = Data_Spliter2(X, Y, folds)
    (foldSize, foldDim) = Xlist[0].shape
    Xt = zeros([foldSize * 4, foldDim])
    Yt = zeros([foldSize * 4, 1])
    Wt = zeros([1, foldDim + 1])
    p = 0
    minEcv = 10000
    for pow in range(-10, 3):
        rate = 10 ** pow
        EcvSum = 0
        for V in range(0, folds):
            beg = 0
            for idx in range(0, folds):
                if idx == V:
                    Xv = Xlist[idx]
                    Yv = Ylist[idx]
                else:
                    Xt[beg: beg + foldSize, :] = Xlist[idx]
                    Ylist[idx].shape = (Ylist[idx].shape[0], 1)
                    Yt[beg: beg + foldSize] = Ylist[idx]
                    beg = beg + foldSize
            W = Ridge_Regression(Xt, Yt, rate)
            Ecv = Err_Counter(Xv, Yv, W)
            EcvSum = EcvSum + Ecv
        if float(EcvSum)/folds <= minEcv:
            minEcv = float(EcvSum)/folds
            (Wt, p) = (W, pow)
    print 'log lambda: ' + str(p)
    print 'Ecv : ' + str(minEcv)
    return (Wt, p)