线性回归（最小二乘法、批量梯度下降法、随机梯度下降法、局部加权线性回归） C++...

向右看齐 2021-09-29 16:24 372阅读 0赞

We turn next to the task of finding a weight vector **w** which minimizes the chosen function E(w).

Because there is clearly no hope of finding an anlytical solution to the equation ∂E(w)=0, we resort to

iterative numerical procedures.

On-line gradient descent, also known as sequential gradient descent or stochastic gradient descent, makes

an update to the weight vector based on one data point at a time.

One advantage of on-line methods compared to batch methods is that the former handle redundancy in the data

much more efficiently. Another property of on-line gradient descent is the possibility of escaping from local minima,

since a stationary point with respect to the error function for the whole data set will generally not be a stationary point

for each data point individually.

Another advantage of on-line learning is the fact that it requires much less storage than batch learning.

[原始数据获得][Link 1]

\#include <iostream>  
\#include <fstream>  
\#include <vector>  
\#include <string>  
\#include <cfloat>  
\#include <cmath>

double dis(double &train, double &query) \{   
　　double weight=exp(-0.5\*pow(train-query, 2));

return weight;  
\}

/\*最小二乘法\*/  
template <typename PairIterator>  
bool GetLinearFit(PairIterator begin\_it, PairIterator end\_it, double& slope, double& y\_intercept) \{   
    if(begin\_it==end\_it) \{   
        return false;  
    \}  
    size\_t n=end\_it-begin\_it;  
    double sum\_x2=0.0,sum\_y=0.0,sum\_x=0.0,sum\_xy=0.0;  
  
    for(PairIterator it=begin\_it;it!=end\_it;++it) \{   
        sum\_x2+=(it->first)\*(it->first);  
        sum\_y+=it->second;  
        sum\_x+=it->first;  
        sum\_xy+=(it->first)\*(it->second);  
    \}  
  
    slope=(n\*sum\_xy-sum\_x\*sum\_y)/(n\*sum\_x2-sum\_x\*sum\_x);  
    y\_intercept=(sum\_x2\*sum\_y-sum\_x\*sum\_xy)/(n\*sum\_x2-sum\_x\*sum\_x);

return true;  
\}

/\*locally weighted linear regression(LWR)\*/  
template<typename PairIterator>  
bool LWR(PairIterator begin\_it, PairIterator end\_it, double& slope, double& y\_intercept) \{   
 　　if(begin\_it==end\_it) \{   
 　　　　return false;  
 　　\}

/\*x are the data points for each local regression model. They are usually (but not always) the data points in your sample.\*/  
 　　double query=5.5;  
 　　size\_t n=end\_it-begin\_it;  
 　　double J=0.0;

for(PairIterator it=begin\_it;it!=end\_it;++it) \{   
 　　　　J+=(y\_intercept+slope\*(it->first)-it->second)\*(y\_intercept+slope\*(it->first)-it->second)\*dis(it->first, query);  
 　　\}  
 　　J=J\*0.5/n;

while(true) \{   
 　　　　double temp0=0,temp1=0;  
 　　　　for(PairIterator it=begin\_it;it!=end\_it;++it) \{   
 　　　　　　temp0+=(y\_intercept+slope\*(it->first)-it->second)\*dis(it->first, query);  
 　　　　　　temp1+=(y\_intercept+slope\*(it->first)-it->second)\*(it->first)\*dis(it->first, query);  
 　　　　\}

temp0=temp0/n;  
 　　　　temp1=temp1/n;

/\*0.03为学习率阿尔法\*/  
 　　　　y\_intercept=y\_intercept-0.03\*temp0;  
 　　　　slope=slope-0.03\*temp1;

double MSE=0.0;  
 　　　　for(PairIterator it=begin\_it;it!=end\_it;++it) \{   
 　　　　　　MSE+=(y\_intercept+slope\*(it->first)-it->second)\*(y\_intercept+slope\*(it->first)-it->second)\*dis(it->first, query);  
 　　　　\}  
 　　　　MSE=0.5\*MSE/n;  
 　　　　if(std::abs(J-MSE)<0.00000001)  
 　　　　　　break;  
 　　　　J=MSE;  
 　　　　\}  
 　　return true;  
\}

/\*批量梯度下降法，Batch Gradient Desscent,BGD\*/  
template<typename PairIterator>  
bool BatchGradientDescent(PairIterator begin\_it, PairIterator end\_it, double& slope, double& y\_intercept) \{   
    if(begin\_it==end\_it) \{   
        return false;  
    \}  
    size\_t n=end\_it-begin\_it;  
    double J=0.0;

while(true) \{   
        double temp0=0,temp1=0;  
        for(PairIterator it=begin\_it;it!=end\_it;++it) \{   
            temp0+=(y\_intercept+slope\*(it->first)-it->second);  
            temp1+=(y\_intercept+slope\*(it->first)-it->second)\*(it->first);  
        \}  
        temp0=temp0/n;  
        temp1=temp1/n;  
  
        /\*0.03为学习率阿尔法\*/  
        y\_intercept=y\_intercept-0.03\*temp0;  
        slope=slope-0.03\*temp1;  
  
        double MSE=0.0;  
        for(PairIterator it=begin\_it;it!=end\_it;++it) \{   
            MSE+=(y\_intercept+slope\*(it->first)-it->second)\*(y\_intercept+slope\*(it->first)-it->second);  
        \}  
        MSE=0.5\*MSE/n;  
        if(std::abs(J-MSE)<0.00000001)  
            break;  
        J=MSE;  
    \}  
    return true;  
\}  
  
/\*随机梯度下降法，Stochastic Gradient Desscent,SGD\*/  
template<typename PairIterator>  
bool StochasticGradientDescent(PairIterator begin\_it, PairIterator end\_it, double& slope, double& y\_intercept) \{   
    if(begin\_it==end\_it) \{   
        return false;  
    \}  
    size\_t n=end\_it-begin\_it;  
    double J=0.0;

/\*the initial cost function\*/  
    for(PairIterator it=begin\_it;it!=end\_it;++it) \{   
        J+=(y\_intercept+slope\*(it->first)-it->second)\*(y\_intercept+slope\*(it->first)-it->second);  
    \}  
    J=0.5\*J/n;  
  
    while(true) \{   
        double temp0=0,temp1=0;  
        for(PairIterator it=begin\_it;it!=end\_it;++it) \{   
            temp0=(y\_intercept+slope\*(it->first)-it->second);  
            temp1=(y\_intercept+slope\*(it->first)-it->second)\*(it->first);  
  
            /\*0.03为学习率阿尔法\*/  
            y\_intercept=y\_intercept-0.03\*temp0;  
            slope=slope-0.03\*temp1;  
  
            double MSE=0.0;  
            for(PairIterator it=begin\_it;it!=end\_it;++it) \{   
                MSE+=(y\_intercept+slope\*(it->first)-it->second)\*(y\_intercept+slope\*(it->first)-it->second);  
            \}  
            MSE=0.5\*MSE/n;  
            if(std::abs(J-MSE)<0.00000001)  
                break;  
            J=MSE;  
        \}  
        break;  
    \}  
  
    return true;  
\}  
  
int main() \{   
    std::ifstream in;  
    in.open("ex2x.dat");  
    if(!in) \{   
        std::cout<<"open file ex2x.dat failed!"<<std::endl;  
        return 1;  
    \}  
  
    std::vector<double> datax,datay;  
    double temp;  
  
    while(in>>temp) \{   
        datax.push\_back(temp);  
    \}  
  
    in.close();  
    in.open("ex2y.dat");  
    if(!in) \{   
        std::cout<<"open file ex2y.dat failed!"<<std::endl;  
        return 1;  
    \}  
  
    while(in>>temp) \{   
        datay.push\_back(temp);  
    \}  
  
    std::vector<std::pair<double, double> > data;  
  
    for(std::vector<double>::const\_iterator iterx=datax.begin(),itery=datay.begin();iterx!=datax.end(),itery!=datay.end();iterx++,itery++) \{   
        data.push\_back(std::pair<double,double>(\*iterx,\*itery));  
    \}  
    in.close();  
    double slope=0.0;  
    double y\_intercept=0.0;  
    GetLinearFit(data.begin(),data.end(),slope,y\_intercept);  
    std::cout<<"最小二乘法得到的结果："<<std::endl;  
    std::cout<<"slope: "<<slope<<std::endl;  
    std::cout<<"y\_intercept: "<<y\_intercept<<std::endl;

slope=1.0,y\_intercept=1.0;  
    BatchGradientDescent(data.begin(),data.end(),slope,y\_intercept);  
    std::cout<<"批量梯度下降法得到的结果："<<std::endl;  
    std::cout<<"slope: "<<slope<<std::endl;  
    std::cout<<"y\_intercept: "<<y\_intercept<<std::endl;  
  
    slope=1.0,y\_intercept=1.0;  
    StochasticGradientDescent(data.begin(),data.end(),slope,y\_intercept);  
    std::cout<<"随机梯度下降法得到的结果："<<std::endl;  
    std::cout<<"slope: "<<slope<<std::endl;  
    std::cout<<"y\_intercept: "<<y\_intercept<<std::endl;

slope=1.0,y\_intercept=1.0;  
 　LWR(data.begin(),data.end(),slope,y\_intercept);  
 　std::cout<<"locally weighted linear regression 得到的结果："<<std::endl;  
 　std::cout<<"slope: "<<slope<<std::endl;  
 　std::cout<<"y\_intercept: "<<y\_intercept<<std::endl;

return 0;  
\}

转载于:https://www.cnblogs.com/donggongdechen/p/7384568.html

[Link 1]: http://openclassroom.stanford.edu/MainFolder/DocumentPage.php?course=MachineLearning&doc=exercises/ex2/ex2.html