Search notes:
Entropy
Entropy measures the uncertainity or information of a random variable.
from numpy import array
from scipy.stats import entropy
from math import log
base = 2
def e(*a):
H = entropy(array(a), base=base)
print(H)
e( 1 ) # 0.0 | maximum value
e( 1/2 , 1/2 ) # 1.0 | maximum value
e( 0.00001 , 0.99999 ) # 0.00018052328301819962 |
e( 1/3 , 1/3 , 1/3 ) # 1.584962500721156 | maximum value
e( 0.00001 , 0.00001 , 0.99998 ) # 0.00036104642176561263
#
# Maximum entropy for 1, 2 and 3 categories:
#
print(log(1, base)) # 0.0
print(log(2, base)) # 1.0
print(log(3, base)) # 1.5849625007211563