This is the respònse of technical support about our question:
Yes, you are right there is a mistake on the manual, unfortunately.
The error is in evaluating parameter cal_b_0 in line
cal_b_0 = Ya - (UInt16) (( (UInt32) cal_m_0 * (UInt32) Xa_ADC0 ) >> 15
);
This equation uses parameter cal_m_0, which is corrected by
CF1_CORRECTION parameter.
So the cal_m_0 and cal_b_0 have to be calculated without correction
parameters and these use in a final value calculating as
Y = (cal_m_0 + CF1_CORRETION)*X + (cal_b_0 + CF2_CORRETION).
Please refer to my code example below:
#define Ya (1023 << 3) // = 8184
#define Yb (3071 << 3) // = 24568
#define DATA_SHEET_PARAMETER_CF1 0.010380
#define DATA_SHEET_PARAMETER_CF2 -31.7
//For "m" correction, need fractional representation of data sheet value
/ 2
#define CF1_CORRECTION (FRAC16(DATA_SHEET_PARAMETER_CF1) * 16384 / 32768 )
// For "b" correction, shift data sheet parameter by 3 bits to allign
with RESULT reg.
#define CF2_CORRECTION ( (INT) ( DATA_SHEET_PARAMETER_CF2 * 8 ) )
static void CalibrateADC_m_and_b(void)
{
Int16 Xa_ADC0, Xb_ADC0, Xa_ADC1, Xb_ADC1;
Xa_ADC0 = low_ADC_cal_0; // = 8304
Xb_ADC0 = high_ADC_cal_0;// = 24600 values measured
Xa_ADC1 = low_ADC_cal_1; // = 8312
Xb_ADC1 = high_ADC_cal_1;// = 24576
cal_m_0 = (UInt16) div_s( ( Yb - Ya) >> 1, ( Xb_ADC0 - Xa_ADC0 ); // = m/2 = 16472
cal_b_0 = Ya - (UInt16) (( (UInt32) cal_m_0 * (UInt32) Xa_ADC0 )
>> 14 );
// =Ya - m*Xa = (1023*8) - [(m/2 + CF1/2)*8304]/16384 = 8184 -
[(16472*8304)/16384] = 8184 - 8348 = -164 //
}
static void CalibrateADCvalue( void )
{
UInt16 ADC_Val_1, ADC_Val_2;
ADC_Val = getReg( ADCA_ADRSLT0 ); // = 9448 value measured
if ( cal_b_0 ) // Have we done calibration yet?
{
// First, multiply by m/2, and shift the result to give the
correct
// bus position.
ADC_Val_1 = (UInt16) ( ((UInt32) (cal_m_0 + CF1_CORRECTION) *
(UInt32) ADC_Val ) >> 14 );
// = [(m/2 +CF1/2)* X]/16384 = (16472+170) * 9448/16384 = 9597
// Then, add the expected offset.
ADC_Val_2 = ( ADC_Val_1 + cal_b_0 + CF2_CORRECTION ) ; //
(9597-164-254) = 9178
ADC_Val = (word) ADC_Val_2;
}
}
This is the way how to obtain the right calibrated value
#define CF1_CORRECTION ((DATA_SHEET_PARAMETER_CF1 / 2) * 32768 )
or
#define CF1_CORRECTION (FRAC16(DATA_SHEET_PARAMETER_CF1 / 2))
both definitions are correct.