Our first humanoid Ml-Agent

This is our second project using Unity ml-agents. To start we need to create a new scene, and this time our character will be Humanoid based.  We are going to use a Space Type Discrete instead of Continuous. And explore a new form to treat the actions from the script.

Our humanoid agent will walk only forward , and we must add more variables in observations, to help him decide when and how to turn.

using System.Collections;
using System.Collections.Generic;
using UnityEngine;
using UnityEngine.UI;
using Unity.MLAgents;
using Unity.MLAgents.Sensors;

public class HumanoidML : Agent
{
    [Header("Speed")]
    [Range(0f, 5f)]
    public float _speed;

    [Header("Turn Speed")]
    [Range(50f, 300f)]
    public float _turnSpeed;

    public bool _training = true;


    private Rigidbody _rb;

    [SerializeField]
    private Transform _target;
    private Animator _anim;

    private Vector3 _previous;


    public override void Initialize()
    {
        _rb = GetComponent<Rigidbody>();
        _anim = GetComponent<Animator>();
        _previous = transform.position;

        //MaxStep is part of Agent Class
        if (!_training) MaxStep = 0;
    }

    public override void OnEpisodeBegin()
    {
        _rb.velocity = Vector3.zero;
        _rb.angularVelocity = Vector3.zero;

        MoverPosicionInicial();
        _previous = transform.position;
    }

    public override void OnActionReceived(float[] vectorAction)
    {
        float lForward = vectorAction[0];
        float lTurn = 0;
        if (vectorAction[1] == 1)
        {
            lTurn = -1;
        }
        else if (vectorAction[1] == 2)
        {
            lTurn = 1;
        }

        _rb.MovePosition(transform.position +
            transform.forward * lForward * _speed * Time.deltaTime);

        transform.Rotate(transform.up * lTurn * _turnSpeed * Time.deltaTime);

    }

    public void Update()
    {
        float velocity = ((transform.position - _previous).magnitude) / Time.deltaTime;
        _previous = transform.position;

        _anim.SetFloat("multiplicador", velocity);
    }

    public override void Heuristic(float[] actionsOut)
    {
        float lForward = 0f;
        float lTurn = 0f;
        if (Input.GetKey(KeyCode.UpArrow))
        {
            lForward = 1f;
        }
        if (Input.GetKey(KeyCode.LeftArrow))
        {
            lTurn = 1f;
        }
        else if (Input.GetKey(KeyCode.RightArrow))
        {
            lTurn = 2f;
        }

        // Put the actions into an array and return
        actionsOut[0] = lForward;
        actionsOut[1] = lTurn;

    }

    public override void CollectObservations(VectorSensor sensor)
    {
        //Distance to the  target.

        sensor.AddObservation(
        Vector3.Distance(_target.transform.position, transform.position));

        //Direction to the target.
        //Vector 3 positions. 
        sensor.AddObservation(
            (_target.transform.position - transform.position).normalized);

        //Forward vector character.
        //Vector 3 positions. 
        sensor.AddObservation(
            transform.forward);
    }

    private void OnTriggerStay(Collider other)
    {
        if (_training)
        {
            if (other.CompareTag("target"))
            {
                AddReward(0.5f);

            }
            if (other.CompareTag("borders"))
            {
                AddReward(-0.05f);
            }
        }
    }


    private void MoverPosicionInicial()
    {
        bool posicionEncontrada = false;
        int intentos = 100;
        Vector3 posicionPotencial = Vector3.zero;

        while (!posicionEncontrada || intentos >= 0)
        {
            intentos--;
            posicionPotencial = new Vector3(
                transform.parent.position.x + UnityEngine.Random.Range(-3f, 3f),
                0.555f,
                transform.parent.position.z + UnityEngine.Random.Range(-3f, 3f));

            Collider[] colliders = Physics.OverlapSphere(posicionPotencial, 0.5f);
            if (colliders.Length == 0)
            {
                transform.position = posicionPotencial;
                posicionEncontrada = true;
            }
        }
    }
}

Files to create the scene:

• Full second scene: https://cursos.uadla.com/wp-content/uploads/2020/08/scene_humanoid.unitypackage.zip

If you don’t want the full scene, and prefer follow alone the video:

• Humanoid character, created with magicavoxel: Humanoid character.
• Low Poly environment pack: https://assetstore.unity.com/packages/3d/environments/landscapes/lowpoly-environment-pack-99479

The commands used to create the second conda environment:

• Create the environment: conda create -n cursoml2 python=3.8
• Activate environment: conda activate cursoml2
• Install numpy: pip install numpy==1.19.0
• Install ml-agents: pip install ml-agents 
• install pytorch: conda install -c pytorch pytorch