#######################################################
# Basic gradient descent alg:
#
# 
#    gradient is the function or any Python callable object 
#    that takes a vector and returns the gradient of the function 
#    you’re trying to minimize.
#
#    start is the point where the algorithm starts its search, 
#    given as a sequence (tuple, list, NumPy array, and so on) 
#    or scalar (in the case of a one-dimensional problem).
#
#    learn_rate is the learning rate that controls the magnitude 
#    of the vector update.
#
#    n_iter is the number of iterations.

def basic_gradient_descent(gradient, start, learn_rate, n_iter):

    vector = start		# Initial estimate point

    for _ in range(n_iter):
        diff = -gradient(vector) * learn_rate	# Delta
        vector += diff				# New estimate

    return vector

##################################################################
# Improved grad desc
#
# The additional parameter tolerance specifies the minimal 
# allowed movement in each iteration.

import numpy as np

def gradient_descent( gradient, start, learn_rate, n_iter=50, tolerance=1e-06):

    vector = start		# Initial estimate point

    for _ in range(n_iter):
        diff = -gradient(vector) * learn_rate   # Delta

        if np.all(np.abs(diff) <= tolerance):	# Exit if movement is small
            break

        vector += diff				# New estimate

    return vector


#######################################################
# Example:
#
#   Scalar functions
#
#     f(x) = x - ln(x)

def gradient1(x):
    return 1 - 1 / x

x =  gradient_descent(gradient1, start=2.5, learn_rate=0.5)
print(x)


#######################################################
# Example:
#
#   Vector functions
#
#     f(v) = v1^2 + v2^4
#     f'   = (2v1, 4v2^3)

def gradient2(v):
    return np.array([2*v[0], 4*v[1]**3])

x =  gradient_descent(gradient2, start=np.array([1.0,1.0]), learn_rate=0.2)
#                                      ^^^^^^^^^^^^^^^^^^^
#                               Cannot use: array([1,1])

print(x)