最速下降优化算法的C++实现

头文件：

/*
 * Copyright (c) 2008-2011 Zhang Ming (M. Zhang), [email protected]
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
 * Free Software Foundation, either version 2 or any later version.
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions are met:
 *
 * 1. Redistributions of source code must retain the above copyright notice,
 *    this list of conditions and the following disclaimer.
 *
 * 2. Redistributions in binary form must reproduce the above copyright
 *    notice, this list of conditions and the following disclaimer in the
 *    documentation and/or other materials provided with the distribution.
 *
 * This program is distributed in the hope that it will be useful, but WITHOUT
 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
 * more details. A copy of the GNU General Public License is available at:
 * http://www.fsf.org/licensing/licenses
 */


/*****************************************************************************
 *                                objfunc.h
 *
 * Function Object.
 *
 * We can use this functor computing the value of objective function and it's
 * gradient vector. The objective function is supposed to be multidimensional,
 * one dimention is the special case of "vector.dim()=1".
 *
 * Zhang Ming, 2010-03, Xi'an Jiaotong University.
 *****************************************************************************/


#ifndef OBJFUNC_H
#define OBJFUNC_H


#include <vector.h>


namespace splab
{

    template <typename Type>
    class ObjFunc
    {

    public:

        /**
         * Initialize the parameters
         */
        ObjFunc( Type aa, Type bb, Type cc ) : a(aa), b(bb), c(cc)
        { }

        /**
         * Compute the value of objective function at point x.
         */
        Type operator()( Vector<Type> &x )
        {
            return ( a*x(1) * exp(b*x(1)*x(1)+c*x(2)*x(2)) );
        }

        /**
         * Compute the gradient of objective function at point x.
         */
        Vector<Type> grad( Vector<Type> &x )
        {
            Type expTerm = exp(a*x(1)*x(1)+b*x(2)*x(2));
            Vector<Type> df(x.dim());
            df(1) = (a+2*a*b*x(1)*x(1)) * expTerm;
            df(2) = 2*a*c*x(1)*x(2) * expTerm;

            return df;
        }

    private:

        // parameters
        Type a,
             b,
             c;

    };
    // class ObjFunc

}
// namespace splab


#endif
// OBJFUNC_H

/*
 * Copyright (c) 2008-2011 Zhang Ming (M. Zhang), [email protected]
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
 * Free Software Foundation, either version 2 or any later version.
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions are met:
 *
 * 1. Redistributions of source code must retain the above copyright notice,
 *    this list of conditions and the following disclaimer.
 *
 * 2. Redistributions in binary form must reproduce the above copyright
 *    notice, this list of conditions and the following disclaimer in the
 *    documentation and/or other materials provided with the distribution.
 *
 * This program is distributed in the hope that it will be useful, but WITHOUT
 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
 * more details. A copy of the GNU General Public License is available at:
 * http://www.fsf.org/licensing/licenses
 */


/*****************************************************************************
 *                               linesearch.h
 *
 * Line search algorithm based on quadratic polynomial interpolation.
 *
 * This routine is often used in orther optimation method, so it is designed
 * as a templated base class.
 *
 * Zhang Ming, 2010-03, Xi'an Jiaotong University.
 *****************************************************************************/


#ifndef LINESEARCH_H
#define LINESEARCH_H


#include <vector.h>


namespace splab
{

    template <typename Dtype, typename Ftype>
    class LineSearch
    {

     public:

        LineSearch();
        ~LineSearch();

        Dtype getStep( Ftype &func, Vector<Dtype> &xk, Vector<Dtype> &dk,
                       int maxItr=10 );

        int getFuncNum() const;
        bool isSuccess() const;

    protected:

        int     funcNum;
        bool    success;

    };
    // class LineSearch


    #include <linesearch-impl.h>

}
// namespace splab


#endif
// LINESEARCH_H

/*
 * Copyright (c) 2008-2011 Zhang Ming (M. Zhang), [email protected]
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
 * Free Software Foundation, either version 2 or any later version.
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions are met:
 *
 * 1. Redistributions of source code must retain the above copyright notice,
 *    this list of conditions and the following disclaimer.
 *
 * 2. Redistributions in binary form must reproduce the above copyright
 *    notice, this list of conditions and the following disclaimer in the
 *    documentation and/or other materials provided with the distribution.
 *
 * This program is distributed in the hope that it will be useful, but WITHOUT
 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
 * more details. A copy of the GNU General Public License is available at:
 * http://www.fsf.org/licensing/licenses
 */


/*****************************************************************************
 *                               steepdesc.h
 *
 * Steepest descent optimal method( gradient algorithm ).
 *
 * This class is designed for finding the minimum value of objective function
 * in one dimension or multidimension. Inexact line search algorithm is used
 * to get the step size in each iteration.
 *
 * Zhang Ming, 2010-03, Xi'an Jiaotong University.
 *****************************************************************************/


#ifndef STEEPDESC_H
#define STEEPDESC_H


#include <linesearch.h>


namespace splab
{

    template <typename Dtype, typename Ftype>
    class SteepDesc : public LineSearch<Dtype, Ftype>
    {

    public:

        SteepDesc();
        ~SteepDesc();

        void optimize( Ftype &func, Vector<Dtype> &x0,
                       Dtype tol=Dtype(1.0e-6), int maxItr=100 );

        Vector<Dtype> getOptValue() const;
        Vector<Dtype> getGradNorm() const;
        Dtype getFuncMin() const;
        int getItrNum() const;

    private:

        // iteration number
        int itrNum;

        // minimum value of objective function
        Dtype fMin;

        // optimal solution
        Vector<Dtype> xOpt;

        // gradient norm for each iteration
        Vector<Dtype> gradNorm;

    };
    // class SteepDesc


    #include <steepdesc-impl.h>

}
// namespace splab


#endif
// STEEPDESC_H

实现文件：

/*
 * Copyright (c) 2008-2011 Zhang Ming (M. Zhang), [email protected]
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
 * Free Software Foundation, either version 2 or any later version.
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions are met:
 *
 * 1. Redistributions of source code must retain the above copyright notice,
 *    this list of conditions and the following disclaimer.
 *
 * 2. Redistributions in binary form must reproduce the above copyright
 *    notice, this list of conditions and the following disclaimer in the
 *    documentation and/or other materials provided with the distribution.
 *
 * This program is distributed in the hope that it will be useful, but WITHOUT
 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
 * more details. A copy of the GNU General Public License is available at:
 * http://www.fsf.org/licensing/licenses
 */


/*****************************************************************************
 *                               linesearch-impl.h
 *
 * Implementation for LineSearch class.
 *
 * Zhang Ming, 2010-03, Xi'an Jiaotong University.
 *****************************************************************************/


/**
 * constructors and destructor
 */
template <typename Dtype, typename Ftype>
LineSearch<Dtype, Ftype>::LineSearch()
{
    funcNum = 0;
    success = true;
}

template <typename Dtype, typename Ftype>
LineSearch<Dtype, Ftype>::~LineSearch()
{
}


/**
 * Finding the step size at point "xk" in direction of "dk" of function
 * "func". The default heuristics number is "maxItr=10".
 */
template <typename Dtype, typename Ftype>
Dtype LineSearch<Dtype, Ftype>::getStep( Ftype &func, Vector<Dtype> &xk,
                                         Vector<Dtype> &dk, int maxItr )
{
    // Set line search parameters that everyone uses.
    Dtype mu = Dtype(0.001),
          kUp = Dtype(0.5),
          kLow = Dtype(0.1),
          alpha = Dtype(1.0),
          alphaMin,
          alphaMax;

    Dtype fNew,
          fk = func(xk);

    Vector<Dtype> xNew,
                  gk = func.grad(xk);

    Dtype gd = dotProd( gk, dk );

    for( int i=0; i<maxItr; ++i )
    {
        xNew = xk + alpha*dk;
        fNew = func(xNew);
        funcNum++;

        if( fNew < fk+mu*alpha*gd )
        {
            success = true;
            return alpha;
        }
        else
        {
            alphaMin = kLow*alpha;
            alphaMax = kUp*alpha;

            // Compute the step by using quadratic polynomial interpolation.
            alpha = Dtype(-0.5)*alpha*alpha*gd / ( fNew-fk-alpha*gd );

            // bound checking
            if( alpha < alphaMin )
                alpha = alphaMin;
            else if( alpha > alphaMax )
                alpha = alphaMax;
        }
    }

    if( fNew>=fk )
    {
        success = false;
        return Dtype(0.0);
    }
    else
    {
        success = true;
        return alpha;
    }
}


/**
 * Get the number of objective function's calculation.
 */
template <typename Dtype, typename Ftype>
inline int LineSearch<Dtype, Ftype>::getFuncNum() const
{
    return funcNum;
}


/**
 * Judgement whether the optimal solution is found or not.
 */
template <typename Dtype, typename Ftype>
inline bool LineSearch<Dtype, Ftype>::isSuccess() const
{
    return success;
}

/*
 * Copyright (c) 2008-2011 Zhang Ming (M. Zhang), [email protected]
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
 * Free Software Foundation, either version 2 or any later version.
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions are met:
 *
 * 1. Redistributions of source code must retain the above copyright notice,
 *    this list of conditions and the following disclaimer.
 *
 * 2. Redistributions in binary form must reproduce the above copyright
 *    notice, this list of conditions and the following disclaimer in the
 *    documentation and/or other materials provided with the distribution.
 *
 * This program is distributed in the hope that it will be useful, but WITHOUT
 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
 * more details. A copy of the GNU General Public License is available at:
 * http://www.fsf.org/licensing/licenses
 */


/*****************************************************************************
 *                               steepdesc-impl.h
 *
 * Implementation for SteepDesc class.
 *
 * Zhang Ming, 2010-03, Xi'an Jiaotong University.
 *****************************************************************************/


/**
 * constructors and destructor
 */
template <typename Dtype, typename Ftype>
SteepDesc<Dtype, Ftype>::SteepDesc() : LineSearch<Dtype, Ftype>()
{
}

template <typename Dtype, typename Ftype>
SteepDesc<Dtype, Ftype>::~SteepDesc()
{
}


/**
 * Finding the optimal solution. The default tolerance error and maximum
 * iteratin number are "tol=1.0e-6" and "maxItr=100", respectively.
 */
template <typename Dtype, typename Ftype>
void SteepDesc<Dtype, Ftype>::optimize( Ftype &func, Vector<Dtype> &x0,
                                        Dtype tol, int maxItr )
{
    // initialize parameters.
    int k = 0;
    Vector<Dtype> x(x0);
    Dtype fx = func(x);
    this->funcNum++;
    Vector<Dtype> gnorm(maxItr);
    Vector<Dtype> g = func.grad(x);
    gnorm[k++]= norm(g);

    Dtype alpha;
    Vector<Dtype> d(g.dim());

    while( ( gnorm(k) > tol ) && ( k < maxItr ) )
    {
        // descent direction
        d =  - g;

        // one dimension searching
        alpha = this->getStep( func, x, d );
        if( !this->success )
            break;

		// update
        x += alpha*d;
        fx = func(x);
        this->funcNum++;
        g = func.grad(x);
        gnorm[k++] = norm(g);
    }

    xOpt = x;
    fMin = fx;
    gradNorm.resize(k);
    for( int i=0; i<k; ++i )
        gradNorm[i] = gnorm[i];

    if( gradNorm[k-1] > tol )
        this->success = false;
}


/**
 * Get the optimum point.
 */
template <typename Dtype, typename Ftype>
inline Vector<Dtype> SteepDesc<Dtype, Ftype>::getOptValue() const
{
    return xOpt;
}


/**
 * Get the norm of gradient in each iteration.
 */
template <typename Dtype, typename Ftype>
inline Vector<Dtype> SteepDesc<Dtype, Ftype>::getGradNorm() const
{
    return gradNorm;
}


/**
 * Get the minimum value of objective function.
 */
template <typename Dtype, typename Ftype>
inline Dtype SteepDesc<Dtype, Ftype>::getFuncMin() const
{
    return fMin;
}


/**
 * Get the iteration number.
 */
template <typename Dtype, typename Ftype>
inline int SteepDesc<Dtype, Ftype>::getItrNum() const
{
    return gradNorm.dim()-1;
}

测试代码：

/*****************************************************************************
 *                               steepdesc_test.cpp
 *
 * Steepest descent method testing.
 *
 * Zhang Ming, 2010-03, Xi'an Jiaotong University.
 *****************************************************************************/


#define BOUNDS_CHECK

#include <iostream>
#include <iomanip>
#include <objfunc.h>
#include <steepdesc.h>


using namespace std;
using namespace splab;


typedef float  Type;


int main()
{
    Type a = 1.0,
         b = -1.0,
         c = -1.0;
    ObjFunc<Type> f( a, b, c );
    Vector<Type> x0(2);
    x0(1) = Type(0.0);
    x0(2) = Type(0.0);

    Type tolErr = 1.0e-3;
    SteepDesc< Type, ObjFunc<Type> > steep;
    steep.optimize( f, x0, tolErr );
    if( steep.isSuccess() )
    {
        Vector<Type> xmin = steep.getOptValue();
        int N = steep.getItrNum();
        cout << "The iterative number is:   " << N << endl << endl;
        cout << "The number of function calculation is:   "
             << steep.getFuncNum() << endl << endl;
        cout << setiosflags(ios::fixed) << setprecision(4);
        cout << "The optimal value of x is:   " << xmin << endl;
        cout << "The minimum value of f(x) is:   " << f(xmin) << endl << endl;
        cout << "The gradient's norm at x is:   "
             << steep.getGradNorm()[N] << endl << endl;
    }
    else
        cout << "The optimal solution  can't be found!" << endl;

    return 0;
}

运行结果：

The iterative number is:   14

The number of function calculation is:   43

The optimal value of x is:   size: 2 by 1
-0.7070
0.0000

The minimum value of f(x) is:   -0.4289

The gradient's norm at x is:   0.0006


Process returned 0 (0x0)   execution time : 0.062 s
Press any key to continue.