Source code for pynir.CalibrationTransfer

# -*- coding: utf-8 -*-
"""
Created on Wed Sep 28 11:00:35 2022

@author: chinn
"""
import numpy as np
from sklearn.linear_model import LinearRegression
from sklearn.cross_decomposition import PLSRegression
from sklearn.decomposition import TruncatedSVD

from scipy.optimize import (BFGS, SR1, Bounds,
                            NonlinearConstraint, minimize)


[docs]class PDS(): def __init__(self, halfWindowSize = 7, regType = 'mlr',**kwargs): self.halfWindowSize = halfWindowSize self.regType = regType self.kwargs = kwargs
[docs] def get_windowRange(halfWindowSize, nFeatrues, i): if i < halfWindowSize: band = np.arange(i) elif i > nFeatrues-halfWindowSize: band = np.arange(i) else: band = np.arange(i-halfWindowSize, i+halfWindowSize) return band
[docs] def fit(self, X1, X2): """ Parameters ---------- X1 : 2D array Standard spectra of master. X2 : 2D array Standard spectra of master. """ if X1.shape != X2.shape: raise("The dimension of two spectral matrix doesn't match.") Models = [] for i in range(X1.shape[1]): if i < self.halfWindowSize: X2i = X2[:,:2*self.halfWindowSize+1] elif i >= self.halfWindowSize and i < X2.shape[1]-self.halfWindowSize: X2i = X2[:,i-self.halfWindowSize:i+self.halfWindowSize+1] elif i > X2.shape[1]-self.halfWindowSize: X2i = X2[:,X2.shape[1]-2*self.halfWindowSize-1:] if self.regType == 'mlr': Models.append(LinearRegression().fit(X2i,X1[:,i][:,None])) elif self.regType == 'pls': Models.append(PLSRegression(n_components=self.n_components).fit(X2i,X1[:,i][:,None])) self.Models = Models return self
[docs] def transform(self, X): Xnew = np.zeros(X.shape) for i in range(X.shape[1]): if i < self.halfWindowSize: Xi = X[:,:2*self.halfWindowSize+1] elif i >= self.halfWindowSize and i < X.shape[1]-self.halfWindowSize: Xi = X[:,i-self.halfWindowSize:i+self.halfWindowSize+1] elif i > X.shape[1]-self.halfWindowSize: Xi = X[:,X.shape[1]-2*self.halfWindowSize-1:] Xnew[:,i] = self.Models[i].predict(Xi).ravel() return Xnew
[docs]class SST(): def __init__(self, n_components=2): self.n_components = n_components
[docs] def fit(self,X1,X2): """ Parameters ---------- X1 : 2D array Standard spectra of the first instrument. X2 : 2D array Standard spectra of the second instrument. """ svd = TruncatedSVD(n_components=self.n_components).fit(np.hstack((X1,X2))) P = svd.components_ nFeature_X1 = X1.shape[1] W1 = P[:,:nFeature_X1] W2 = P[:,nFeature_X1:] self.F = np.dot(np.linalg.pinv(W2), (W1-W2)) return self
[docs] def transform(self,X): return X + np.dot(X,self.F)
[docs]class BS(): """ Ref Osborne B. G., Fearn T. Collaborative evaluation of universal calibrations for the measurement of protein and moisture in flour by near infrared reflectance [J]. International Journal of Food Science & Technology, 1983, 18(4): 453-460. """ def __init__(self): pass
[docs] def fit(self, y1, y2): self.lrModel = LinearRegression().fit(y2.reshape(-1,1), y1.reshape(-1,1)) return self
[docs] def transform(self, y2): return self.lrModel.predict(y2.reshape(-1,1))
[docs]class NS_PFCE(): def __init__(self,thres = 0.98, constrType = 1): self.thres = thres self.constrType = constrType
[docs] def fit(self, X1, X2, b1): ntask = 2 lb = -np.inf ub = 0.0 if self.constrType == 1: nlc = NonlinearConstraint( lambda b2:pfce_constr1(np.hstack((b1.reshape(-1,1), b2.reshape(-1,1))), self.thres), lb, ub) elif self.constrType == 2: nlc = NonlinearConstraint( lambda b2:pfce_constr2(np.hstack((b1.reshape(-1,1), b2.reshape(-1,1))), self.thres), lb, ub) elif self.constrType == 3: nlc = NonlinearConstraint( lambda b2:pfce_constr3(np.hstack((b1.reshape(-1,1), b2.reshape(-1,1))), self.thres), lb, ub) b2 = minimize(lambda b2:cost_NS_PFCE(b1,b2,X1,X2), x0 = b1, method='SLSQP', constraints=[nlc]) self.b2 = b2 return self
[docs] def transform(self, X): return np.dot(np.hstack((np.ones([X.shape[0],1]), X)), np.reshape(self.b2.x,(-1,1)))
[docs]class SS_PFCE(): def __init__(self,thres = 0.98, constrType = 1): self.thres = thres self.constrType = constrType
[docs] def fit(self, X2, y, b1): ntask = 2 lb = -np.inf ub = 0.0 if self.constrType == 1: nlc = NonlinearConstraint( lambda b2:pfce_constr1(np.hstack((b1.reshape(-1,1), b2.reshape(-1,1))), self.thres), lb, ub) elif self.constrType == 2: nlc = NonlinearConstraint( lambda b2:pfce_constr2(np.hstack((b1.reshape(-1,1), b2.reshape(-1,1))), self.thres), lb, ub) elif self.constrType == 3: nlc = NonlinearConstraint( lambda b2:pfce_constr3(np.hstack((b1.reshape(-1,1), b2.reshape(-1,1))), self.thres), lb, ub) else: raise("PFCE only support the constrint type of 1, 2, or 3") b2 = minimize(lambda b2:cost_SS_PFCE(b2,X2, np.ravel(y)), x0 = b1, method='SLSQP', constraints=[nlc]) self.b2 = b2 return self
[docs] def transform(self, X): return np.dot(np.hstack((np.ones([X.shape[0],1]), X)), np.reshape(self.b2.x,(-1,1)))
[docs]class FS_PFCE(): def __init__(self,thres = 0.98, constrType = 1): self.thres = thres self.constrType = constrType
[docs] def fit(self, X1, X2, y, b1): ntask = 2 lb = -np.inf ub = 0.0 if self.constrType == 1: nlc = NonlinearConstraint( lambda b2:pfce_constr1(np.hstack((b1.reshape(-1,1), b2.reshape(-1,1))), self.thres), lb, ub) elif self.constrType == 2: nlc = NonlinearConstraint( lambda b2:pfce_constr2(np.hstack((b1.reshape(-1,1), b2.reshape(-1,1))), self.thres), lb, ub) elif self.constrType == 3: nlc = NonlinearConstraint( lambda b2:pfce_constr3(np.hstack((b1.reshape(-1,1), b2.reshape(-1,1))), self.thres), lb, ub) b2 = minimize(lambda b2:cost_FS_PFCE(b1,b2,X1,X2,np.ravel(y)), x0 = b1, method='SLSQP', constraints=[nlc]) self.b2 = b2 return self
[docs] def transform(self, X): return np.dot(np.hstack((np.ones([X.shape[0],1]), X)), np.reshape(self.b2.x,(-1,1)))
[docs]class MT_PFCE(): def __init__(self,thres = 0.98, constrType = 1): self.thres = thres self.constrType = constrType
[docs] def fit(self, X, y, b1): ntask = len(X) y = [np.ravel(yi) for yi in y] self.ntask = ntask lb = -np.inf ub = 0.0 if self.constrType == 1: nlc = NonlinearConstraint( lambda B:pfce_constr1(np.transpose(np.reshape(B,(ntask,-1))), self.thres), lb, ub) elif self.constrType == 2: nlc = NonlinearConstraint( lambda B:pfce_constr2(np.transpose(np.reshape(B,(ntask,-1))), self.thres), lb, ub) elif self.constrType == 3: nlc = NonlinearConstraint( lambda B:pfce_constr3(np.transpose(np.reshape(B,(ntask,-1))), self.thres), lb, ub) B = minimize(lambda B:cost_MT_PFCE(B.reshape(ntask,-1).transpose(),X,y), x0 = np.tile(b1,ntask),method='SLSQP',constraints=[nlc]) self.B = B return self
[docs] def transform(self, X, itask): return np.dot(np.hstack((np.ones([X.shape[0],1]), X)), ((self.B.x).reshape(self.ntask,-1).transpose()[:,itask]).reshape((-1,1)))
[docs]def cost_NS_PFCE(b1, b2, X1, X2): yhat1 = np.dot(np.hstack((np.ones([X1.shape[0],1]), X1)), b1) yhat2 = np.dot(np.hstack((np.ones([X2.shape[0],1]), X2)), b2) error = yhat1-yhat2 return np.sum(error**2)
[docs]def cost_SS_PFCE(b2, X2, y): yhat2 = np.dot(np.hstack((np.ones([X2.shape[0],1]), X2)), b2) error = y-yhat2 return np.sum(error**2)
[docs]def cost_FS_PFCE(b1, b2, X1, X2, y): yhat1 = np.dot(np.hstack((np.ones([X1.shape[0],1]), X1)), b1) yhat2 = np.dot(np.hstack((np.ones([X2.shape[0],1]), X2)), b2) error1 = yhat1-yhat2 error2 = y-yhat2 return np.sum(error1**2/len(error1)+error2**2/len(error2))
[docs]def cost_MT_PFCE(B, X, y): Error = [] for i in range(B.shape[1]): yhati = np.dot(np.hstack((np.ones([X[i].shape[0],1]), X[i])), B[:,i]) errori = y[i]-yhati Error.append(np.mean(errori**2)) return np.sum(Error)
[docs]def pfce_constr1(B,thres): corr = np.corrcoef((B[1:,].transpose())) constr_cost = thres - np.sum(np.triu(corr,1))/np.sum(np.arange(B.shape[1])) return constr_cost
[docs]def pfce_constr2(B,thres): B = B[1:,] Vars = np.diag(np.dot(B.transpose(),B)) VarErr = 0 for i in range(B.shape[1]): Err = B-B[:,i][:,None] Erri = Err; Erri = np.delete(Erri, i, axis =1) Varsi = Vars Varsi = np.delete(Varsi, i) VarErr += np.sum(Erri**2,axis = 0)/np.sqrt(Vars[i]*Varsi) dist = VarErr/(B.shape[1] * (B.shape[1]-1)) constr_cost = thres - 1 + dist return constr_cost
[docs]def pfce_constr3(B,thres): B = B[1:,] norm1s = np.sum(np.abs(B),axis=0) norm1Err = 0 for i in range(B.shape[1]): Err = B-B[:,i][:,None] Erri = Err Erri = np.delete(Erri, i,axis = 1) norm1si = norm1s norm1si = np.delete(norm1si, i) norm1Err += np.sum(np.abs(Erri),axis = 0)/np.sqrt(norm1s[i]*norm1si) dist = norm1Err/(B.shape[1] * (B.shape[1]-1)) constr_cost = thres - 1 + dist return constr_cost