diff --git a/src/ntops/kernels/rms_norm.py b/src/ntops/kernels/rms_norm.py index 797314c..f4959e2 100644 --- a/src/ntops/kernels/rms_norm.py +++ b/src/ntops/kernels/rms_norm.py @@ -9,12 +9,13 @@ def application(input, weight, eps, output, num_normalized_elements): _rms = ntl.zeros(input.dtype.shape, dtype=ntl.float32) + epsilon = ntl.cast(eps, ntl.float32) for i in range(input.shape[0]): input_i = ntl.cast(input[i], ntl.float32) _rms += input_i * input_i - rms = ntl.sqrt(ntl.sum(_rms) / num_normalized_elements + eps) + rms = ntl.sqrt(ntl.sum(_rms) / num_normalized_elements + epsilon) for i in range(input.shape[0]): output[i] = input[i] / rms * weight[i]