RLModule.h

Go to the documentation of this file.

/************************************************************************
* Verve                                                                 *
* Copyright (C) 2004-2006                                               *
* Tyler Streeter  tylerstreeter@gmail.com                               *
* All rights reserved.                                                  *
* Web: http://verve-agents.sourceforge.net                              *
*                                                                       *
* This library is free software; you can redistribute it and/or         *
* modify it under the terms of EITHER:                                  *
*   (1) The GNU Lesser General Public License as published by the Free  *
*       Software Foundation; either version 2.1 of the License, or (at  *
*       your option) any later version. The text of the GNU Lesser      *
*       General Public License is included with this library in the     *
*       file license-LGPL.txt.                                          *
*   (2) The BSD-style license that is included with this library in     *
*       the file license-BSD.txt.                                       *
*                                                                       *
* This library is distributed in the hope that it will be useful,       *
* but WITHOUT ANY WARRANTY; without even the implied warranty of        *
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the files    *
* license-LGPL.txt and license-BSD.txt for more details.                *
************************************************************************/

#ifndef VERVE_RL_MODULE_H
#define VERVE_RL_MODULE_H

#include "Globals.h"
#include "RBFInputData.h"
#include "ActiveTDConnectionList.h"

namespace verve
{
        class Observation;
        class Population;
        class Projection;
        class RBFPopulation;
        class WinnerTakeAllPopulation;

        class RLModule
        {
        public:
                VERVE_DECL RLModule(const Observation& obs, 
                        bool isDynamicRBFEnabled, unsigned int numActions);

                VERVE_DECL virtual ~RLModule();

                VERVE_DECL virtual void VERVE_CALL resetShortTermMemory();

                VERVE_DECL virtual unsigned int VERVE_CALL update(
                        const Observation& obs, real reinforcement);

                VERVE_DECL virtual unsigned int VERVE_CALL updatePolicyOnly(
                        const Observation& obs);

                VERVE_DECL virtual void VERVE_CALL changeStepSize(real newValue);

                VERVE_DECL virtual void VERVE_CALL setETraceTimeConstant(
                        real timeConstant, real stepSize);

                VERVE_DECL virtual void VERVE_CALL setTDDiscountTimeConstant(
                        real timeConstant, real stepSize);

                VERVE_DECL virtual void VERVE_CALL setTDLearningRate(
                        real valueFunctionTimeConstant, real policyLearningMultiplier, 
                        real stepSize);

                VERVE_DECL virtual real VERVE_CALL getTDError();

                VERVE_DECL virtual void VERVE_CALL resetState(const Observation& obs);

                VERVE_DECL virtual real VERVE_CALL computeValueEstimation(
                        const Observation& obs);

                VERVE_DECL virtual void VERVE_CALL saveValueData(
                        unsigned int continuousResolution, 
                        const std::string& filename = "");

                VERVE_DECL virtual void VERVE_CALL saveStateRBFData(
                        const std::string& filename = "");

        protected:
                void updateActiveTDConnectionList();

                void trainTDRule();

                real updateCriticOutput();

                unsigned int updateActorOutput();

                RBFInputData mLatestInputData;

                RBFPopulation* mStateRepresentation;

                WinnerTakeAllPopulation* mActorPopulation;

                Population* mCriticPopulation;

                std::vector<Population*> mAllPopulations;

                ActiveTDConnectionList mActiveValueFunctionTDConnections;

                ActiveTDConnectionList mActivePolicyTDConnections;

                bool mFirstStep;

                real mTDError;

                real mOldValueEstimation;

                real mNewValueEstimation;

                real mETraceTimeConstant;

                real mTDDiscountTimeConstant;

                real mTDDiscountFactor;

                real mValueFunctionLearningTimeConstant;

                real mValueFunctionLearningFactor;

                real mPolicyLearningMultiplier;
        };
}

#endif

---