Estrategia de Señal Keltner de Aprendizaje por Refuerzo

using System;
using System.Collections.Generic;
using System.Linq;

using Ecng.Common;

using StockSharp.Algo.Indicators;
using StockSharp.Algo.Strategies;
using StockSharp.BusinessEntities;
using StockSharp.Messages;

namespace StockSharp.Samples.Strategies;

/// <summary>
/// Keltner breakouts filtered by an online, one-hidden-layer neural Q learner.
/// </summary>
public class KeltnerWithRLSignalStrategy : Strategy
{
	private readonly StrategyParam<int> _emaPeriod;
	private readonly StrategyParam<int> _atrPeriod;
	private readonly StrategyParam<decimal> _atrMultiplier;
	private readonly StrategyParam<decimal> _stopLossAtr;
	private readonly StrategyParam<int> _cooldownBars;
	private readonly StrategyParam<DataType> _candleType;
	private readonly StrategyParam<double> _learningRate;
	private readonly StrategyParam<double> _discountFactor;
	private readonly StrategyParam<double> _exploration;
	private readonly StrategyParam<int> _randomSeed;

	private double[] _previousFeatures;
	private int _previousAction;
	private decimal _previousPrice;
	private decimal _previousAtr;
	private decimal _entryPrice;
	private int _cooldownRemaining;
	private bool _previousAboveUpperBand;
	private bool _previousBelowLowerBand;
	private Order _pendingOrder;

	public int EmaPeriod { get => _emaPeriod.Value; set => _emaPeriod.Value = value; }
	public int AtrPeriod { get => _atrPeriod.Value; set => _atrPeriod.Value = value; }
	public decimal AtrMultiplier { get => _atrMultiplier.Value; set => _atrMultiplier.Value = value; }
	public decimal StopLossAtr { get => _stopLossAtr.Value; set => _stopLossAtr.Value = value; }
	public int CooldownBars { get => _cooldownBars.Value; set => _cooldownBars.Value = value; }
	public DataType CandleType { get => _candleType.Value; set => _candleType.Value = value; }
	public double LearningRate { get => _learningRate.Value; set => _learningRate.Value = value; }
	public double DiscountFactor { get => _discountFactor.Value; set => _discountFactor.Value = value; }
	public double Exploration { get => _exploration.Value; set => _exploration.Value = value; }
	public int RandomSeed { get => _randomSeed.Value; set => _randomSeed.Value = value; }

	// Read-only model diagnostics; learned weights are deliberately not strategy settings.
	public NeuralQModel LearningModel { get; private set; }
	public int CurrentSignal { get; private set; }

	public KeltnerWithRLSignalStrategy()
	{
		_emaPeriod = Param(nameof(EmaPeriod), 20).SetGreaterThanZero()
			.SetDisplay("EMA Period", "Period for the exponential moving average", "Keltner Settings").SetOptimize(10, 30, 5);
		_atrPeriod = Param(nameof(AtrPeriod), 14).SetGreaterThanZero()
			.SetDisplay("ATR Period", "Period for the average true range", "Keltner Settings").SetOptimize(7, 21, 7);
		_atrMultiplier = Param(nameof(AtrMultiplier), 2m).SetGreaterThanZero()
			.SetDisplay("ATR Multiplier", "Multiplier for ATR in Keltner Channels", "Keltner Settings").SetOptimize(1.5m, 3m, 0.5m);
		_stopLossAtr = Param(nameof(StopLossAtr), 2m).SetGreaterThanZero()
			.SetDisplay("Stop Loss (ATR)", "Stop Loss in multiples of ATR", "Risk Management").SetOptimize(1m, 3m, 0.5m);
		_cooldownBars = Param(nameof(CooldownBars), 48).SetNotNegative()
			.SetDisplay("Cooldown Bars", "Closed candles to wait before another position change", "General");
		_candleType = Param(nameof(CandleType), TimeSpan.FromMinutes(15).TimeFrame())
			.SetDisplay("Candle Type", "Type of candles to use", "General");
		_learningRate = Param(nameof(LearningRate), 0.05).SetNotNegative();
		_discountFactor = Param(nameof(DiscountFactor), 0.9).SetNotNegative();
		_exploration = Param(nameof(Exploration), 0.1).SetNotNegative();
		_randomSeed = Param(nameof(RandomSeed), 42).SetNotNegative();
	}

	public override IEnumerable<(Security sec, DataType dt)> GetWorkingSecurities()
		=> [(Security, CandleType)];

	public NeuralQModel CreateLearningModel()
		=> new(RandomSeed, LearningRate, DiscountFactor, Exploration);

	protected override void OnReseted()
	{
		base.OnReseted();
		ResetState();
	}

	private void ResetState()
	{
		LearningModel = null;
		CurrentSignal = 0;
		_previousFeatures = null;
		_previousAction = 0;
		_previousPrice = _previousAtr = _entryPrice = 0m;
		_cooldownRemaining = 0;
		_previousAboveUpperBand = _previousBelowLowerBand = false;
		_pendingOrder = null;
	}

	protected override void OnStarted2(DateTime time)
	{
		base.OnStarted2(time);
		ResetState();
		LearningModel = CreateLearningModel();
		var ema = new ExponentialMovingAverage { Length = EmaPeriod };
		var atr = new AverageTrueRange { Length = AtrPeriod };
		var subscription = SubscribeCandles(CandleType);
		subscription.Bind(ema, atr, ProcessCandle).Start();

		var area = CreateChartArea();
		if (area != null)
		{
			DrawCandles(area, subscription);
			DrawIndicator(area, ema);
			DrawOwnTrades(area);
		}
	}

	private void ProcessCandle(ICandleMessage candle, decimal middleBand, decimal atr)
	{
		if (candle.State != CandleStates.Finished || !IsFormedAndOnlineAndAllowTrading() || atr <= 0m)
			return;

		var price = candle.ClosePrice;
		double[] features =
		[
			Math.Tanh((double)((price - middleBand) / atr)),
			_previousPrice == 0m ? 0.0 : Math.Tanh((double)((price - _previousPrice) / atr)),
			_previousAtr == 0m ? 0.0 : Math.Tanh((double)((atr - _previousAtr) / _previousAtr)),
			Math.Tanh((double)((price - candle.OpenPrice) / atr)),
		];

		// Reward the preceding policy action only after its next close is known.
		// This is a hypothetical one-bar return, not a fill-price PnL estimate.
		if (_previousFeatures != null)
		{
			var direction = _previousAction == 1 ? 1.0 : _previousAction == 2 ? -1.0 : 0.0;
			var reward = Math.Clamp(direction * (double)((price - _previousPrice) / _previousAtr), -1.0, 1.0);
			LearningModel.Learn(_previousFeatures, _previousAction, reward, features);
		}
		CurrentSignal = LearningModel.SelectAction(features);
		_previousFeatures = features;
		_previousAction = CurrentSignal;
		_previousPrice = price;
		_previousAtr = atr;

		if (_cooldownRemaining > 0)
			_cooldownRemaining--;
		if (_pendingOrder?.State is OrderStates.Done or OrderStates.Failed)
			_pendingOrder = null;
		if (Position == 0m)
			_entryPrice = 0m;

		var above = price > middleBand + AtrMultiplier * atr;
		var below = price < middleBand - AtrMultiplier * atr;
		var buy = !_previousAboveUpperBand && above && CurrentSignal == 1;
		var sell = !_previousBelowLowerBand && below && CurrentSignal == 2;
		_previousAboveUpperBand = above;
		_previousBelowLowerBand = below;

		if (_pendingOrder != null)
			return;

		// One order per callback: a reversal must not also submit an EMA/stop exit.
		if (_cooldownRemaining == 0 && buy && Position <= 0m)
			Submit(Sides.Buy, Volume + Math.Abs(Position), price);
		else if (_cooldownRemaining == 0 && sell && Position >= 0m)
			Submit(Sides.Sell, Volume + Math.Abs(Position), price);
		else if (Position > 0m && (price < middleBand || (_entryPrice > 0m && price < _entryPrice - StopLossAtr * atr)))
			Submit(Sides.Sell, Math.Abs(Position), 0m);
		else if (Position < 0m && (price > middleBand || (_entryPrice > 0m && price > _entryPrice + StopLossAtr * atr)))
			Submit(Sides.Buy, Math.Abs(Position), 0m);
	}

	private void Submit(Sides side, decimal volume, decimal entryPrice)
	{
		_entryPrice = entryPrice;
		_cooldownRemaining = CooldownBars;
		_pendingOrder = side == Sides.Buy ? BuyMarket(volume) : SellMarket(volume);
	}

	/// <summary>
	/// Four inputs, eight tanh hidden neurons and three linear Q outputs:
	/// neutral, buy and sell. Both layers learn with a detached TD target.
	/// </summary>
	public sealed class NeuralQModel
	{
		private readonly double[][] _hidden = new double[8][];
		private readonly double[][] _output = new double[3][];
		private readonly double _learningRate;
		private readonly double _discount;
		private readonly double _exploration;
		private uint _randomState;

		public int Updates { get; private set; }

		public NeuralQModel(int seed, double learningRate, double discount, double exploration)
		{
			if (seed < 0)
				throw new ArgumentOutOfRangeException(nameof(seed));
			if (!double.IsFinite(learningRate) || learningRate < 0.0 || learningRate > 1.0)
				throw new ArgumentOutOfRangeException(nameof(learningRate));
			if (!double.IsFinite(discount) || discount < 0.0 || discount > 1.0)
				throw new ArgumentOutOfRangeException(nameof(discount));
			if (!double.IsFinite(exploration) || exploration < 0.0 || exploration > 1.0)
				throw new ArgumentOutOfRangeException(nameof(exploration));
			_learningRate = learningRate;
			_discount = discount;
			_exploration = exploration;
			_randomState = (uint)seed;
			for (var i = 0; i < 8; i++)
				_hidden[i] = Enumerable.Range(0, 5).Select(_ => (NextRandom() - 0.5) * 0.2).ToArray();
			for (var i = 0; i < 3; i++)
				_output[i] = Enumerable.Range(0, 9).Select(_ => (NextRandom() - 0.5) * 0.2).ToArray();
		}

		private double NextRandom()
		{
			_randomState = unchecked(1664525u * _randomState + 1013904223u);
			return _randomState / 4294967296.0;
		}

		private double[] HiddenValues(double[] features)
		{
			if (features.Length != 4 || features.Any(value => !double.IsFinite(value)))
				throw new ArgumentException("Four finite features are required.", nameof(features));
			var values = new double[8];
			for (var neuron = 0; neuron < 8; neuron++)
			{
				var sum = _hidden[neuron][4];
				for (var feature = 0; feature < 4; feature++)
					sum += _hidden[neuron][feature] * features[feature];
				values[neuron] = Math.Tanh(sum);
			}
			return values;
		}

		public double[] Predict(double[] features)
		{
			var hidden = HiddenValues(features);
			var result = new double[3];
			for (var action = 0; action < 3; action++)
			{
				result[action] = _output[action][8];
				for (var neuron = 0; neuron < 8; neuron++)
					result[action] += _output[action][neuron] * hidden[neuron];
			}
			return result;
		}

		public int SelectAction(double[] features)
		{
			var values = Predict(features);
			if (NextRandom() < _exploration)
				return (int)(NextRandom() * 3);
			var best = 0;
			for (var action = 1; action < 3; action++)
				if (values[action] > values[best])
					best = action;
			return best;
		}

		public void Learn(double[] state, int action, double reward, double[] nextState)
		{
			if (action < 0 || action > 2 || !double.IsFinite(reward))
				throw new ArgumentOutOfRangeException(nameof(action));
			var hidden = HiddenValues(state);
			var error = Math.Clamp(reward + _discount * Predict(nextState).Max() - Predict(state)[action], -1.0, 1.0);
			if (_learningRate == 0.0)
				return;
			// Backprop uses the output weights from BEFORE their update.
			var previousOutput = (double[])_output[action].Clone();
			for (var neuron = 0; neuron < 8; neuron++)
			{
				_output[action][neuron] += _learningRate * error * hidden[neuron];
				var gradient = error * previousOutput[neuron] * (1.0 - hidden[neuron] * hidden[neuron]);
				for (var feature = 0; feature < 4; feature++)
					_hidden[neuron][feature] += _learningRate * gradient * state[feature];
				_hidden[neuron][4] += _learningRate * gradient;
			}
			_output[action][8] += _learningRate * error;
			Updates++;
		}

		public double[] GetWeights()
			=> _hidden.Concat(_output).SelectMany(layer => layer).ToArray();
	}
}

import clr

clr.AddReference("StockSharp.Messages")
clr.AddReference("StockSharp.Algo")
clr.AddReference("StockSharp.Algo.Indicators")
clr.AddReference("StockSharp.Algo.Strategies")

from System import TimeSpan, Math, Decimal, Array, Double
from StockSharp.Messages import DataType, CandleStates, Sides, OrderStates
from StockSharp.Algo.Indicators import ExponentialMovingAverage, AverageTrueRange
from StockSharp.Algo.Strategies import Strategy


class NeuralQModel:
    """Four inputs, eight trainable tanh hidden neurons, three linear Q outputs."""

    def __init__(self, seed, learning_rate, discount, exploration):
        if int(seed) < 0:
            raise ValueError("RandomSeed must be nonnegative")
        for value in (learning_rate, discount, exploration):
            if not Double.IsFinite(float(value)) or value < 0 or value > 1:
                raise ValueError("LearningRate, DiscountFactor and Exploration must be in [0, 1]")
        self._learning_rate = float(learning_rate)
        self._discount = float(discount)
        self._exploration = float(exploration)
        self._random_state = int(seed)
        self.Updates = 0
        self._hidden = [[(self._next_random() - 0.5) * 0.2 for _ in range(5)] for _ in range(8)]
        self._output = [[(self._next_random() - 0.5) * 0.2 for _ in range(9)] for _ in range(3)]

    def _next_random(self):
        # Identical unsigned 32-bit generator in C# and Python.
        self._random_state = (1664525 * self._random_state + 1013904223) & 0xffffffff
        return self._random_state / 4294967296.0

    def _hidden_values(self, features):
        if len(features) != 4 or any(not Double.IsFinite(float(value)) for value in features):
            raise ValueError("Four finite features are required")
        values = []
        for neuron in range(8):
            total = self._hidden[neuron][4]
            for feature in range(4):
                total += self._hidden[neuron][feature] * float(features[feature])
            values.append(float(Math.Tanh(total)))
        return values

    def Predict(self, features):
        hidden = self._hidden_values(features)
        values = []
        for action in range(3):
            total = self._output[action][8]
            for neuron in range(8):
                total += self._output[action][neuron] * hidden[neuron]
            values.append(total)
        return Array[Double](values)

    def SelectAction(self, features):
        values = self.Predict(features)
        if self._next_random() < self._exploration:
            return int(self._next_random() * 3)
        best = 0
        for action in range(1, 3):
            if values[action] > values[best]:
                best = action
        return best

    def Learn(self, state, action, reward, next_state):
        action = int(action)
        reward = float(reward)
        if action < 0 or action > 2 or not Double.IsFinite(reward):
            raise ValueError("A valid action and finite reward are required")
        hidden = self._hidden_values(state)
        error = max(-1.0, min(1.0, reward + self._discount * max(self.Predict(next_state)) - self.Predict(state)[action]))
        if self._learning_rate == 0:
            return
        previous_output = list(self._output[action])
        for neuron in range(8):
            self._output[action][neuron] += self._learning_rate * error * hidden[neuron]
            gradient = error * previous_output[neuron] * (1.0 - hidden[neuron] * hidden[neuron])
            for feature in range(4):
                self._hidden[neuron][feature] += self._learning_rate * gradient * float(state[feature])
            self._hidden[neuron][4] += self._learning_rate * gradient
        self._output[action][8] += self._learning_rate * error
        self.Updates += 1

    def GetWeights(self):
        return Array[Double]([value for layer in self._hidden + self._output for value in layer])


class keltner_with_rl_signal_strategy(Strategy):
    """Keltner breakouts filtered by an online neural Q learner."""

    def __init__(self):
        super(keltner_with_rl_signal_strategy, self).__init__()
        self._ema_period = self.Param("EmaPeriod", 20).SetGreaterThanZero().SetDisplay("EMA Period", "Period for the exponential moving average", "Keltner Settings")
        self._atr_period = self.Param("AtrPeriod", 14).SetGreaterThanZero().SetDisplay("ATR Period", "Period for the average true range", "Keltner Settings")
        self._atr_multiplier = self.Param("AtrMultiplier", 2.0).SetGreaterThanZero().SetDisplay("ATR Multiplier", "Multiplier for ATR in Keltner Channels", "Keltner Settings")
        self._stop_loss_atr = self.Param("StopLossAtr", 2.0).SetGreaterThanZero().SetDisplay("Stop Loss (ATR)", "Stop Loss in multiples of ATR", "Risk Management")
        self._cooldown_bars = self.Param("CooldownBars", 48).SetNotNegative().SetDisplay("Cooldown Bars", "Closed candles to wait before another position change", "General")
        self._candle_type = self.Param("CandleType", DataType.TimeFrame(TimeSpan.FromMinutes(15))).SetDisplay("Candle Type", "Type of candles to use", "General")
        self._learning_rate = self.Param("LearningRate", 0.05).SetNotNegative()
        self._discount_factor = self.Param("DiscountFactor", 0.9).SetNotNegative()
        self._exploration = self.Param("Exploration", 0.1).SetNotNegative()
        self._random_seed = self.Param("RandomSeed", 42).SetNotNegative()
        self._reset_state()

    def _reset_state(self):
        self.LearningModel = None
        self.CurrentSignal = 0
        self._previous_features = None
        self._previous_action = 0
        self._previous_price = Decimal(0)
        self._previous_atr = Decimal(0)
        self._entry_price = Decimal(0)
        self._cooldown_remaining = 0
        self._previous_above_upper = False
        self._previous_below_lower = False
        self._pending_order = None

    def CreateLearningModel(self):
        return NeuralQModel(int(self._random_seed.Value), float(self._learning_rate.Value),
                            float(self._discount_factor.Value), float(self._exploration.Value))

    @property
    def candle_type(self):
        return self._candle_type.Value

    def GetWorkingSecurities(self):
        return [(self.Security, self.candle_type)]

    def OnReseted(self):
        super(keltner_with_rl_signal_strategy, self).OnReseted()
        self._reset_state()

    def OnStarted2(self, time):
        super(keltner_with_rl_signal_strategy, self).OnStarted2(time)
        self._reset_state()
        self.LearningModel = self.CreateLearningModel()
        ema = ExponentialMovingAverage()
        ema.Length = int(self._ema_period.Value)
        atr = AverageTrueRange()
        atr.Length = int(self._atr_period.Value)
        subscription = self.SubscribeCandles(self.candle_type)
        subscription.Bind(ema, atr, self.ProcessCandle).Start()
        area = self.CreateChartArea()
        if area is not None:
            self.DrawCandles(area, subscription)
            self.DrawIndicator(area, ema)
            self.DrawOwnTrades(area)

    def ProcessCandle(self, candle, middle_band, atr):
        if candle.State != CandleStates.Finished or not self.IsFormedAndOnlineAndAllowTrading() or atr <= 0:
            return
        price = candle.ClosePrice
        features = Array[Double]([
            float(Math.Tanh(float((price - middle_band) / atr))),
            0.0 if self._previous_price == 0 else float(Math.Tanh(float((price - self._previous_price) / atr))),
            0.0 if self._previous_atr == 0 else float(Math.Tanh(float((atr - self._previous_atr) / self._previous_atr))),
            float(Math.Tanh(float((price - candle.OpenPrice) / atr))),
        ])
        # Only the next completed bar supplies the reward for the preceding action.
        # Hypothetical one-bar return, not actual fill PnL.
        if self._previous_features is not None:
            direction = 1.0 if self._previous_action == 1 else -1.0 if self._previous_action == 2 else 0.0
            reward = max(-1.0, min(1.0, direction * float((price - self._previous_price) / self._previous_atr)))
            self.LearningModel.Learn(self._previous_features, self._previous_action, reward, features)
        self.CurrentSignal = self.LearningModel.SelectAction(features)
        self._previous_features = features
        self._previous_action = self.CurrentSignal
        self._previous_price = price
        self._previous_atr = atr

        if self._cooldown_remaining > 0:
            self._cooldown_remaining -= 1
        if self._pending_order is not None and self._pending_order.State in (OrderStates.Done, OrderStates.Failed):
            self._pending_order = None
        if self.Position == 0:
            self._entry_price = Decimal(0)

        offset = Decimal(self._atr_multiplier.Value) * atr
        above = price > middle_band + offset
        below = price < middle_band - offset
        buy = not self._previous_above_upper and above and self.CurrentSignal == 1
        sell = not self._previous_below_lower and below and self.CurrentSignal == 2
        self._previous_above_upper = above
        self._previous_below_lower = below
        if self._pending_order is not None:
            return
        stop_offset = Decimal(self._stop_loss_atr.Value) * atr
        if self._cooldown_remaining == 0 and buy and self.Position <= 0:
            self._submit(Sides.Buy, self.Volume + Math.Abs(self.Position), price)
        elif self._cooldown_remaining == 0 and sell and self.Position >= 0:
            self._submit(Sides.Sell, self.Volume + Math.Abs(self.Position), price)
        elif self.Position > 0 and (price < middle_band or (self._entry_price > 0 and price < self._entry_price - stop_offset)):
            self._submit(Sides.Sell, Math.Abs(self.Position), Decimal(0))
        elif self.Position < 0 and (price > middle_band or (self._entry_price > 0 and price > self._entry_price + stop_offset)):
            self._submit(Sides.Buy, Math.Abs(self.Position), Decimal(0))

    def _submit(self, side, volume, entry_price):
        self._entry_price = entry_price
        self._cooldown_remaining = int(self._cooldown_bars.Value)
        self._pending_order = self.BuyMarket(volume) if side == Sides.Buy else self.SellMarket(volume)

    def CreateClone(self):
        return keltner_with_rl_signal_strategy()