use truncation instead of rounding down or rounding to the nearest value when numerical calculations exceed precision limits

This commit is contained in:
MaysWind
2025-09-09 20:46:51 +08:00
parent d4603a1892
commit 7e2e1a4ad3
13 changed files with 36 additions and 36 deletions
+3 -3
View File
@@ -42,12 +42,12 @@ function formatCoordinateValue(value: number, positiveDirectionName: string, neg
if (displayFormat === CoordinateDisplayFormat.DecimalDegrees) {
return `${prefix}${value.toFixed(6)}${suffix}`;
} else if (displayFormat === CoordinateDisplayFormat.DecimalMinutes) {
const degrees = Math.floor(value);
const degrees = Math.trunc(value);
const minutes = (value - degrees) * 60;
return `${prefix}${degrees}°${minutes.toFixed(5)}'${suffix}`;
} else if (displayFormat === CoordinateDisplayFormat.DegreesMinutesSeconds) {
const degrees = Math.floor(value);
const minutes = Math.floor((value - degrees) * 60);
const degrees = Math.trunc(value);
const minutes = Math.trunc((value - degrees) * 60);
const seconds = (value - degrees - minutes / 60) * 3600;
return `${prefix}${degrees}°${minutes}'${seconds.toFixed(4)}"${suffix}`;
} else {
+3 -3
View File
@@ -35,7 +35,7 @@ function normalizeNumber(textualNumber: string): number {
}
function denormalizeNumberToAmount(num: number): number {
return Math.floor(num / normalizedNumberToAmountFactor);
return Math.trunc(num / normalizedNumberToAmountFactor);
}
function checkNumberRange(num: number): void {
@@ -180,14 +180,14 @@ function evaluatePostfixExpr(tokens: string[]): number | null {
result = a - b;
break;
case '*':
result = Math.floor(a * b / normalizeFactor);
result = Math.trunc(a * b / normalizeFactor);
break;
case '/':
if (b === 0) {
logger.warn(`cannot evaluate expression "${tokens.join(' ')}", because division by zero`);
return null;
}
result = Math.floor(a * normalizeFactor / b);
result = Math.trunc(a * normalizeFactor / b);
break;
default:
return null;
+4 -4
View File
@@ -230,7 +230,7 @@ export function formatNumber(value: number, options: NumberFormatOptions, precis
if (isDefined(precision)) {
const ratio = Math.pow(10, precision);
const normalizedValue = Math.floor(value * ratio);
const normalizedValue = Math.trunc(value * ratio);
const textualValue = numeralSystem.formatNumber(normalizedValue / ratio);
return appendDigitGroupingSymbolAndDecimalSeparator(textualValue, options);
} else {
@@ -242,7 +242,7 @@ export function formatNumber(value: number, options: NumberFormatOptions, precis
export function formatPercent(value: number, precision: number, lowPrecisionValue: string, options: NumberFormatOptions): string {
const numeralSystem = options.numeralSystem || NumeralSystem.Default;
const ratio = Math.pow(10, precision);
const normalizedValue = Math.floor(value * ratio);
const normalizedValue = Math.trunc(value * ratio);
if (value > 0 && normalizedValue < 1 && lowPrecisionValue) {
const systemDecimalSeparator = DecimalSeparator.Dot.symbol;
@@ -262,9 +262,9 @@ export function formatPercent(value: number, precision: number, lowPrecisionValu
export function getAmountWithDecimalNumberCount(amount: number, decimalNumberCount: number): number {
if (decimalNumberCount === 0) {
return Math.floor(amount / 100) * 100;
return Math.trunc(amount / 100) * 100;
} else if (decimalNumberCount === 1) {
return Math.floor(amount / 10) * 10;
return Math.trunc(amount / 10) * 10;
}
return amount;