# -*- coding: utf-8 -*-
r"""
imageprocessing module is made to modify filter data array
More tools can be found in LaueTools package at sourceforge.net and gitlab.esrf.fr
March 2020
"""
__author__ = "Jean-Sebastien Micha, CRG-IF BM32 @ ESRF"
# built-in modules
import sys
import os
# third party modules
# import scipy.interpolate as sci
import scipy.ndimage as ndimage
import scipy.signal
import scipy.spatial.distance as ssd
try:
from PIL import Image
PIL_EXISTS = True
except ImportError:
print("Missing python module called PIL. Please install it if you need open some tiff "
"images from vhr camera")
PIL_EXISTS = False
import numpy as np
import pylab as pp
# lauetools modules
if sys.version_info.major == 3:
from . import generaltools as GT
from . import dict_LaueTools as DictLT
else:
import generaltools as GT
import dict_LaueTools as DictLT
listfile = os.listdir(os.curdir)
[docs]def getindices2cropArray(center, halfboxsizeROI, arrayshape, flipxycenter=False):
r"""
return array indices limits to crop array data
:param center: iterable of 2 elements
(x,y) pixel center of the ROI
:param halfboxsizeROI: integer or iterable of 2 elements
half boxsize ROI in two dimensions
:param arrayshape: iterable of 2 integers
maximal number of pixels in both directions
:param flipxycenter: boolean
True: swap x and y of center with respect to others
parameters that remain fixed
:return: imin, imax, jmin, jmax : 4 integers
4 indices allowing to slice a 2D np.ndarray
.. todo:: merge with check_array_indices()
"""
xpic, ypic = center
if flipxycenter:
ypic, xpic = center
xpic, ypic = int(xpic), int(ypic)
if isinstance(halfboxsizeROI, int):
boxsizex, boxsizey = halfboxsizeROI, halfboxsizeROI
else:
boxsizex, boxsizey = halfboxsizeROI
x1 = np.maximum(0, xpic - boxsizex)
x2 = np.minimum(arrayshape[0], xpic + boxsizex)
y1 = np.maximum(0, ypic - boxsizey)
y2 = np.minimum(arrayshape[1], ypic + boxsizey)
imin, imax, jmin, jmax = y1, y2, x1, x2
return imin, imax, jmin, jmax
[docs]def check_array_indices(imin, imax, jmin, jmax, framedim=None):
r"""
Return 4 indices for array slice compatible with framedim
:param imin, imax, jmin, jmax: 4 integers
mini. and maxi. indices in both directions
:param framedim: iterable of 2 integers
shape of the array to be sliced by means of the 4 indices
:return: imin, imax, jmin, jmax: 4 integers
mini. and maxi. indices in both directions
.. todo:: merge with getindices2cropArray()
"""
if framedim is None:
print("framedim is empty in check_array_indices()")
return
imin = max(imin, 0)
jmin = max(jmin, 0)
imax = min(framedim[0], imax)
jmax = min(framedim[1], jmax)
return imin, imax, jmin, jmax
### Modify images
[docs]def to8bits(PILimage, normalization_value=None):
r"""
convert PIL image (16 bits) in 8 bits PIL image
:return: - [0] 8 bits image
- [1] corresponding pixels value array
.. todo:: since not used, may be deleted
"""
imagesize = PILimage.size
image8bits = Image.new("L", imagesize)
rawdata = np.array(PILimage.getdata())
if not normalization_value:
normalization_value = 1.0 * np.amax(rawdata)
datatoput = np.array(rawdata / normalization_value * 255, dtype="uint8")
image8bits.putdata(datatoput)
return image8bits, datatoput
# --- ------------- getting data from images or ROI
[docs]def diff_pix(pix, array_pix, radius=1):
r"""
returns
index in array_pix which is the closest to pix if below the tolerance radius
array_pix: array of 2d pixel points
pix: one 2elements pixel point
"""
dist2 = sum((pix - array_pix) ** 2, axis=1)
closepix = np.argmin(dist2)
if closepix:
if dist2[closepix] <= radius ** 2:
return closepix
else:
return None
[docs]def minmax(D_array, center, boxsize, framedim=(2048, 2048), withmaxpos=False):
r"""
extract min and max from a 2d array in a ROI
Obsolete? Still used in LocalMaxima_ShiftArrays()
Parameters
D_array : 2D array
data array
center : iterable of 2 integers
(x,y) pixel center
boxsize : integer or iterable of 2 integers
full boxsize defined in both directions
framedim : iterable of 2 integers
shape of D_array
Return
[min, max]: minimium and maximum pixel internsity in ROI
[min, max],absolute_max_pos : if withmaxpos is True add in output
the absolute position of the largest pixel
#TODO: replace by scipy.ndimage.extrema
# see next functions below
# framedim = from dictionary of CCDs
D_array shape is flip(framedim)
"""
if not isinstance(boxsize, int):
boxsize = (boxsize, boxsize)
# print "framedim in minmax", framedim
# print "D_array.shape", D_array.shape
# halfbox = int(boxsize / 2)
# xc, yc = center
# imin, imax, jmin, jmax = max(0, yc - halfbox), \
# min(yc + halfbox, framedim[0]), \
# max(0, xc - halfbox), \
# min(framedim[1], xc + halfbox)
#
# print "imin, imax, jmin, jmax", imin, imax, jmin, jmax
framedim = framedim[1], framedim[0]
imin, imax, jmin, jmax = getindices2cropArray(center, boxsize, framedim, flipxycenter=1)
# print "imin, imax, jmin, jmax", imin, imax, jmin, jmax
fulldata = D_array
array_short = fulldata[imin:imax, jmin:jmax]
mini_in_ROI = np.amin(array_short)
maxi_in_ROI = np.amax(array_short)
absolute_max_pos = ndimage.maximum_position(array_short) + np.array([imin, jmin])
if withmaxpos:
return [mini_in_ROI, maxi_in_ROI], absolute_max_pos
else:
return [mini_in_ROI, maxi_in_ROI]
[docs]def getExtrema(data2d, center, boxsize, framedim, ROIcoords=0, flipxycenter=True):
r"""
return min max XYposmin, XYposmax values in ROI
:param ROIcoords: 1 in local array indices coordinates
0 in X,Y pixel CCD coordinates
:param flipxycenter: boolean like
swap input center coordinates
:param data2d: 2D array
data array as read by :func:`readCCDimage`
:return: min, max, XYposmin, XYposmax:
- min : minimum pixel intensity
- max : maximum pixel intensity
- XYposmin : list of absolute pixel coordinates of lowest pixel
- XYposmax : list of absolute pixel coordinates of largest pixel
"""
if center is None or len(center) == 0:
raise ValueError("center (peak list) in getExtrema is empty")
indicesborders = getindices2cropArray(center, [boxsize, boxsize], framedim,
flipxycenter=flipxycenter)
imin, imax, jmin, jmax = indicesborders
print("imin, imax, jmin, jmax", imin, imax, jmin, jmax)
datacropped = data2d[imin:imax, jmin:jmax]
# mini, maxi, posmin, posmax
mini, maxi, posmin, posmax = ndimage.measurements.extrema(datacropped)
if ROIcoords:
return mini, maxi, posmin, posmax
else:
max_i, max_j = posmax
min_i, min_j = posmin
if flipxycenter:
centery, centerx = center
else:
centerx, centery = center
# print "local position of maximum i,j", max_i, max_j
globalXmax = max_j + centerx - boxsize
globalYmax = max_i + centery - boxsize
globalXmin = min_j + centerx - boxsize
globalYmin = min_i + centery - boxsize
# print "Highest intensity %.f at (X,Y): (%d,%d) " % (maxi, globalX, globalY)
if flipxycenter:
return mini, maxi, [globalYmin, globalXmin], [globalYmax, globalXmax]
else:
return mini, maxi, [globalXmin, globalYmin], [globalXmax, globalYmax]
[docs]def getIntegratedIntensity(data2d, center, boxsize, framedim,
thresholdlevel=0.2, flipxycenter=True):
r"""
return crude estimate of integrated intensity of peak above a given relative threshold
:param ROIcoords: 1 in local array indices coordinates
0 in X,Y pixel CCD coordinates
:param flipxycenter: boolean like
swap input center coordinates
:param data2d: 2D array
data array as read by :func:`readCCDimage`
:param Thresholdlevel: relative level above which pixel intensity must be taken into account
I(p)- minimum> Thresholdlevel* (maximum-minimum)
:return: integrated intensity, minimum absolute intensity, nbpixels used for the summation
"""
if not center:
raise ValueError("center (peak list) in getExtrema is empty")
indicesborders = getindices2cropArray(center, [boxsize, boxsize], framedim, flipxycenter=flipxycenter)
imin, imax, jmin, jmax = indicesborders
# print "imin, imax, jmin, jmax", imin, imax, jmin, jmax
datacropped = data2d[imin:imax, jmin:jmax]
# mini, maxi, posmin, posmax
mini, maxi, _, _ = ndimage.measurements.extrema(datacropped)
minimum_amplitude = thresholdlevel * (maxi - mini) + mini
print("integration for pixel intensity higher than: ", minimum_amplitude)
pixelsabove = datacropped[datacropped > minimum_amplitude]
nbpixels = len(pixelsabove)
print("nb pixels above threshold ", nbpixels)
return ndimage.measurements.sum(pixelsabove), minimum_amplitude, nbpixels
[docs]def getMinMax(data2d, center, boxsize, framedim):
r"""
return min and max values in ROI
Parameters:
data2d : 2D array
array as read by readCCDimage
"""
return getExtrema(data2d, center, boxsize, framedim)[:2]
[docs]def minmax_fast(D_array, centers, boxsize=(25, 25)):
r"""
extract min (considered as background in boxsize) and intensity at center
from a 2d array at different places (centers)
centers is tuple a two array ( array([slow indices]), array([fast indices]))
return:
[0] background values
[1] intensity value
used?
"""
min_array = ndimage.minimum_filter(D_array, size=boxsize)
return [min_array[centers], D_array[centers]]
# --- ------------- Mexican Hat 2D kernel
def myfromfunction(f, s, t):
return np.fromfunction(f, s).astype(t)
[docs]def normalize_shape(shape):
r"""
return shape
in case a scalar was given:
return (shape,)
"""
try:
len(shape)
return shape
except TypeError:
return (shape,)
[docs]def LoG(r, sigma=None, dim=1, r0=None, peakVal=None):
r"""note:
returns *negative* Laplacian-of-Gaussian (aka. mexican hat)
zero-point will be at sqrt(dim)*sigma
integral is _always_ 0
if peakVal is None: uses "mathematical" "gaussian derived" norm
if r0 is not None: specify radius of zero-point (IGNORE sigma !!)
"""
r2 = r ** 2
if sigma is None:
if r0 is not None:
sigma = float(r0) / np.sqrt(dim)
else:
raise ValueError("One of sigma or r0 have to be non-None")
else:
if r0 is not None:
raise ValueError("Only one of sigma or r0 can be non-None")
s2 = sigma ** 2
dsd = dim * sigma ** dim
if peakVal is not None:
norm = peakVal / dsd
else:
norm = 1.0 / (s2 * (2.0 * np.pi * sigma) ** (dim / 2.0))
return np.exp(-r2 / (2.0 * s2)) * (dsd - r2) * norm
[docs]def LoGArr(shape=(256, 256),
r0=None,
sigma=None,
peakVal=None,
orig=None,
wrap=0,
dtype=np.float32):
r"""returns n-dim Laplacian-of-Gaussian (aka. mexican hat)
if peakVal is not None
result max is peakVal
if r0 is not None: specify radius of zero-point (IGNORE sigma !!)
credits: "Sebastian Haase <haase@msg.ucsf.edu>"
"""
shape = normalize_shape(shape)
dim = len(shape)
return radialArr(shape,
lambda r: LoG(r, sigma=sigma, dim=dim, r0=r0, peakVal=peakVal),
orig,
wrap,
dtype)
[docs]def radialArr(shape, func, orig=None, wrap=False, dtype=np.float32):
r"""generates and returns radially symmetric function sampled in volume(image) of shape shape
if orig is None the origin defaults to the center
func is a 1D function with 1 paramater: r
if shape is a scalar uses implicitely `(shape,)`
wrap tells if functions is continued wrapping around image boundaries
wrap can be True or False or a tuple same length as shape:
then wrap is given for each axis sperately
"""
shape = normalize_shape(shape)
try:
len(shape)
except TypeError:
shape = (shape,)
if orig is None:
orig = (np.array(shape, dtype=np.float) - 1) / 2.0
else:
try:
oo = float(orig)
orig = np.ones(shape=len(shape)) * oo
except:
pass
if len(shape) != len(orig):
raise ValueError("shape and orig not same dimension")
try:
if len(wrap) != len(shape):
raise ValueError("wrap tuple must be same length as shape")
except TypeError:
wrap = (wrap,) * len(shape)
def wrapIt(ax, q):
if wrap[ax]:
nq = shape[ax]
return np.where(q > nq // 2, q - nq, q)
else:
return q
# if wrap:
# def wrapIt(q, nq):
# return np.where(q>nq/2,q-nq, q)
# else:
# def wrapIt(q, nq):
# return q
# if len(shape) == 1:
# x0 = orig[0] # 20060606: [0] prevents orig (as array) promoting its dtype (e.g. Float64) into result
# nx = shape[0]
# return myfromfunction(lambda x: func(wrapIt(np.absolute(x-x0),nx)), shape, dtype)
# elif len(shape) == 2:
# y0,x0 = orig
# ny,nx=shape
# return myfromfunction(lambda y,x: func(
# np.sqrt( \
# (wrapIt((x-x0),nx))**2 + (wrapIt((y-y0),ny))**2 ) ), shape, dtype)
# elif len(shape) == 3:
# z0,y0,x0 = orig
# nz,ny,nx=shape
# return myfromfunction(lambda z,y,x: func(
# np.sqrt( \
# (wrapIt((x-x0),nx))**2 + (wrapIt((y-y0),ny))**2 + (wrapIt((z-z0),nz))**2 ) ), shape, dtype)
if len(shape) == 1:
x0 = orig[0] # 20060606: [0] prevents orig (as array) promoting its dtype (e.g. Float64) into result
return myfromfunction(lambda x: func(wrapIt(0, np.absolute(x - x0))), shape, dtype)
elif len(shape) == 2:
y0, x0 = orig
return myfromfunction(lambda y, x: func(
np.sqrt((wrapIt(-1, x - x0)) ** 2 + (wrapIt(-2, y - y0)) ** 2)
), shape, dtype)
elif len(shape) == 3:
z0, y0, x0 = orig
return myfromfunction(lambda z, y, x: func(
np.sqrt((wrapIt(-1, x - x0)) ** 2
+ (wrapIt(-2, y - y0)) ** 2
+ (wrapIt(-3, z - z0)) ** 2)), shape, dtype)
else:
raise ValueError("only defined for dim < 3 (#TODO)")
# --- -------------------- Local Maxima or Local Hot pixels search
[docs]def LocalMaxima_ndimage(Data,
peakVal=4,
boxsize=5,
central_radius=2,
threshold=1000,
connectivity=1,
returnfloatmeanpos=0,
autothresholdpercentage=None):
r"""
returns (float) i,j positions in array of each blob
(peak, spot, assembly of hot pixels or whatever)
.. note:: used only in LocalMaxima_KernelConvolution
inputs
peakVal, boxsize, central_radius:
parameters for numerical convolution with a mexican-hat-like kernel
threshold:
intensity threshold of filtered Data (by convolution with the kernel)
above which blob signal will be considered
if = 0 : take all blobs at the expense of processing time
connectivity :
1 for filled square 3*3 connectivity
0 for 3*3 star like connectivity
autothresholdpercentage :
threshold in filtered image with respect to the maximum intensity in filtered image
output:
array (n,2): array of 2 indices
"""
aa = ConvolvebyKernel(Data, peakVal=peakVal, boxsize=boxsize, central_radius=central_radius)
print("Histogram after convolution with Mexican Hat")
print(np.histogram(aa))
if autothresholdpercentage is None:
thraa = np.where(aa > threshold, 1, 0)
else:
thraa = np.where(aa > (autothresholdpercentage / 100.0) * np.amax(aa), 1, 0)
if connectivity == 0:
star = np.eye(3)
ll, nf = ndimage.label(thraa, structure=star)
elif connectivity == 1:
ll, nf = ndimage.label(thraa, structure=np.ones((3, 3)))
elif connectivity == 2:
ll, nf = ndimage.label(thraa, structure=np.array([[1, 1, 1], [0, 1, 0], [1, 1, 1]]))
elif connectivity == 3:
ll, nf = ndimage.label(thraa, structure=np.array([[0, 1, 0], [1, 1, 1], [0, 1, 0]]))
# meanpos = np.array(ndimage.measurements.center_of_mass(thraa,
# ll,
# np.arange(1, nf + 1)),
# dtype=np.float)
meanpos = np.array(ndimage.measurements.maximum_position(thraa, ll, np.arange(1, nf + 1)),
dtype=np.float)
if returnfloatmeanpos:
return meanpos
else:
return np.array(meanpos, dtype=np.int)
[docs]def ConvolvebyKernel(Data, peakVal=4, boxsize=5, central_radius=2):
r"""
Convolve Data array witn mexican-hat kernel
inputs:
Data : 2D array containing pixel intensities
peakVal > central_radius : defines pixel distance from box center where weights are positive
(in the middle) and negative farther to converge back to zero
boxsize : size of the box
ouput:
array (same shape as Data)
"""
# from scipy import ndimage
# outa = np.zeros(Data.shape)
# ndimage.filters.gaussian_laplace(d,(5,5),output=outa)
# whole_structure= createstructure(10, 10)-2*createstructure(10, 7)+4*createstructure(10, 5)
# bb= ndimage.convolve(d,whole_structure)
# bb=ndimage.morphology.white_tophat(Data,(boxsize,boxsize))
# mexicanhat = array(LoGArr((10,10),r0=6,peakVal=4),dtype= int16)
mexicanhat = LoGArr((boxsize, boxsize), r0=central_radius, peakVal=peakVal)
mexicanhat = mexicanhat - sum(mexicanhat) / mexicanhat.size
bb = ndimage.convolve(np.array(Data, dtype=np.float32), mexicanhat)
return bb
[docs]def LocalMaxima_KernelConvolution(Data, framedim=(2048, 2048),
peakValConvolve=4, boxsizeConvolve=5, central_radiusConvolve=2,
thresholdConvolve=1000,
connectivity=1,
IntensityThreshold=500,
boxsize_for_probing_minimal_value_background=30,
return_nb_raw_blobs=0,
peakposition_definition="max"): # full side length
r"""
return local maxima (blobs) position and amplitude in Data by using
convolution with a mexican hat like kernel.
Two Thresholds are used sequently:
- thresholdConvolve : level under which intensity of kernel-convolved array is discarded
- IntensityThreshold : level under which blob whose local intensity amplitude in raw array is discarded
:param Data: 2D array containing pixel intensities
:param peakValConvolve, boxsizeConvolve, central_radiusConvolve: convolution kernel parameters
:param thresholdConvolve: minimum threshold (expressed in unit of convolved array intensity)
under which convoluted blob is rejected.It can be zero
(all blobs are accepted but time consuming)
:param connectivity: shape of connectivity pattern to consider pixels belonging to the
same blob.
- 1: filled square (1 pixel connected to 8 neighbours)
- 0: star (4 neighbours in vertical and horizontal direction)
:param IntensityThreshold: minimum local blob amplitude to accept
:param boxsize_for_probing_minimal_value_background: boxsize to evaluate the background
and the blob amplitude
:param peakposition_definition: string ('max' or 'center')
key to assign to the blob position its hottest pixel position
or its center (no weight)
:return:
peakslist : array like (n,2)
list of peaks position (pixel)
Ipixmax : array like (n,1) of integer
list of highest pixel intensity in the vicinity of each peak
npeaks : integer
nb of peaks (if return_nb_raw_blobs =1)
"""
print("framedim in LocalMaxima_KernelConvolution", framedim)
print("Data.shape", Data.shape)
dataimage_ROI = Data
peak = LocalMaxima_ndimage(dataimage_ROI,
peakVal=peakValConvolve,
boxsize=boxsizeConvolve,
central_radius=central_radiusConvolve,
threshold=thresholdConvolve,
connectivity=connectivity,
returnfloatmeanpos=0)
if not peak:
return None
peak = (peak[:, 0], peak[:, 1])
intensity_localmaxima = dataimage_ROI[peak]
peaki = peak[0]
peakj = peak[1]
# building an array of hot pixels (2 coordinates)
Yarray = peakj
Xarray = peaki
peaklist = np.array([Xarray, Yarray]).T
# print "peaklist", peaklist
# print "%d local maxima have been found from convolution method" % len(peaklist)
# print "peaklist[:3]", peaklist[:3]
# probing background and maximal intensity in boxsize
#
ptp_boxsize = boxsize_for_probing_minimal_value_background
# first method ---------------------------
tabptp = [] # tab of min and max around each peak
tabposmax = [] # # tab of position of hottest pixel close to that found after convolution
for k in list(range(len(peaklist))):
# print "k in LocalMaxima_KernelConvolution", k
# print "dataimage_ROI.shape", dataimage_ROI.shape
minimaxi, maxpos = minmax(dataimage_ROI, peaklist[k], ptp_boxsize, framedim=framedim, withmaxpos=1)
tabptp.append(minimaxi)
# if minimaxi[1] > 4000:
# print "k, peaklist[k]", k, peaklist[k]
# print maxpos, minimaxi[1]
# mini, maxi, minpos, maxpos = getExtrema(dataimage_ROI, peaklist[k], ptp_boxsize, framedim,
# ROIcoords=0,
# flipxycenter=1)
#
# tabptp.append([mini, maxi])
#
# if maxi > 4000:
# print "k, peaklist[k]", k, peaklist[k]
# print maxpos, maxi
tabposmax.append(maxpos) # # new
# # to test peaks position / framedim
# print minmax(dataimage_ROI,
# [2580, 2750],
# ptp_boxsize,
# framedim=framedim,
# withmaxpos=1)
ar_ptp = np.array(tabptp)
ar_posmax = np.array(tabposmax) # # new
# print ' ar_posmax[:3]', ar_posmax[:10]
# print ' ar_posmax[:3]', ar_posmax[10:20]
# print ' ar_posmax[:3]', ar_posmax[20:30]
# print "saturation at", np.where(ar_ptp > 65000.)
# -------------------------------------------
# second method ---------------
# tabptp = minmax_fast(dataimage_ROI,
# tuple(transpose(peaklist)),
# boxsize=(boxsize_for_probing_minimal_value_background,
# boxsize_for_probing_minimal_value_background))
# ar_ptp = array(tabptp).T
# ---------------------------------
# ar_amp = np.subtract(ar_ptp[:,1],ar_ptp[:,0])
ar_amp = np.subtract(intensity_localmaxima, ar_ptp[:, 0])
amp_rank = np.argsort(ar_amp)[::-1]
peaklist_sorted = peaklist[amp_rank]
ptp_sorted = ar_ptp[amp_rank]
amp_sorted = ar_amp[amp_rank]
posmax_sorted = ar_posmax[amp_rank] # # new
# thresholding on peak-to-peak amplitude
threshold_amp = IntensityThreshold
cond = np.where(amp_sorted > threshold_amp)
th_peaklist = peaklist_sorted[cond]
th_ar_ptp = ptp_sorted[cond]
th_ar_amp = amp_sorted[cond]
th_ar_pos = posmax_sorted[cond] # # new
# print "th_ar_ptp", th_ar_ptp[:10]
##### peak positions that will be returned are the hottest pixels
if peakposition_definition == "max":
th_peaklist = th_ar_pos
else:
# using th_peaklist which is float position
pass
print("{} local maxima found after thresholding above {} (amplitude above local background)".format(
len(th_ar_amp), threshold_amp))
# NEW --- from method array shift!
# remove duplicates (close points), the most intense pixel is kept
# minimum distance between hot pixel
# it corresponds both to distance between peaks and peak size ...
pixeldistance = 10 # pixeldistance_remove_duplicates
purged_pklist, index_todelete = GT.purgeClosePoints2(th_peaklist, pixeldistance)
purged_amp = np.delete(th_ar_amp, index_todelete)
purged_ptp = np.delete(th_ar_ptp, index_todelete, axis=0)
# print 'shape of purged_ptp method conv.', purged_ptp.shape
print("{} local maxima found after removing duplicates (minimum intermaxima distance = {})".format(
len(purged_amp), pixeldistance))
# print "purged_pklist", purged_pklist
# print "shape(purged_pklist)", np.shape(purged_pklist)
npeaks = np.shape(purged_pklist)[0]
# print np.shape(Data)
Ipixmax = purged_ptp[:, 1]
# print "Ipixmax = ", Ipixmax
peakslist = np.fliplr(purged_pklist)
if return_nb_raw_blobs:
return peakslist, Ipixmax, npeaks
else:
return peakslist, Ipixmax
# NEW --- !
# -----------------------
# npeaks = np.shape(th_peaklist)[0]
# Ipixmax = np.zeros(npeaks, dtype=int)
# # print np.shape(Data)
# for i in list(range(npeaks)):
# # Ipixmax[i]=Data[th_peaklist[i,0],th_peaklist[i,1]]
# Ipixmax[i] = th_ar_ptp[i][1]
#
# if return_nb_raw_blobs == 1:
# return np.fliplr(th_peaklist), Ipixmax, len(peaklist)
# else:
# return np.fliplr(th_peaklist), Ipixmax
[docs]def LocalMaxima_ShiftArrays(Data, framedim=(2048, 2048), IntensityThreshold=500,
Saturation_value=65535,
boxsize_for_probing_minimal_value_background=30, # full side length
nb_of_shift=25,
pixeldistance_remove_duplicates=25,
verbose=0):
r""" blob search or local maxima search by shift array method (kind of derivative)
.. warning:: Flat peak (= two neighbouring pixel with rigourouslty the same intensity)
is not detected
"""
try:
import networkx as NX
except ImportError:
print("\n***********************************************************")
print("networkx module is missing! Some functions may not work...\nPlease install it at http://networkx.github.io/")
print("***********************************************************\n")
xminfit2d, xmaxfit2d, yminfit2d, ymaxfit2d = (1, framedim[1], 1, framedim[0])
# warning i corresponds to y
# j corresponds to x
# change nom xminf2d => xminfit2d pour coherence avec le reste
imin, _, jmin, _ = (framedim[0] - ymaxfit2d,
framedim[0] - yminfit2d,
xminfit2d - 1,
xmaxfit2d - 1)
# dataimage_ROI=dataimage[imin:imax,jmin:jmax]# array index i,j
# # fit2d index: X=j Y=2048-i
dataimage_ROI = Data
print("searching local maxima for non saturated consecutive pixels")
peak = localmaxima(dataimage_ROI, nb_of_shift, diags=1)
print("Done...!")
print(peak)
# print "execution time : %f secondes"%(ttt.time()-time_0)
intensity_localmaxima = dataimage_ROI[peak]
# print intensity_localmaxima
# SATURATION handling ------------------------------
# if the top of the local maximum has at least two pixels with the same intensity, this maximum is not detected
# this generally the case for saturated peaks
# saturation value : saturation above which we will take into account the pixel
# this value may be lower than the 2^n bits value to handle unfortunately very flat weak peak with 2 neighbouring pixels
# Saturation_value = 65535 for mccd
Size_of_pixelconnection = 20
print("Saturation value for flat top peak handling", Saturation_value)
sat_pix = np.where(dataimage_ROI >= Saturation_value)
if verbose:
print("positions of saturated pixels \n", sat_pix)
print("nb of saturated pixels", len(sat_pix[0]))
sat_pix_mean = None
# there is at least one peak above or equal to the Saturation_value threshold
# loop over saturated pixels
if len(sat_pix[0]) > 0:
if verbose:
print("positions of saturated pixels \n", sat_pix)
# use of graph algorithms
sat_pix = np.column_stack(sat_pix)
disttable_sat = ssd.pdist(sat_pix, "euclidean")
sqdistmatrix_sat = ssd.squareform(disttable_sat)
# building adjencymat
a, b = np.indices(sqdistmatrix_sat.shape)
indymat = np.triu(b) + np.tril(a)
cond2 = np.logical_and(sqdistmatrix_sat < Size_of_pixelconnection, sqdistmatrix_sat > 0)
adjencymat = np.where(cond2, indymat, 0)
# print "before networkx"
print("networkx version", NX.__version__)
GGraw = NX.to_networkx_graph(adjencymat, create_using=NX.Graph())
list_of_cliques = NX.find_cliques(GGraw)
# print "after networkx"
# now find average pixel of each clique
sat_pix_mean = []
for clique in list_of_cliques:
ii, jj = np.mean(sat_pix[clique], axis=0)
sat_pix_mean.append([int(ii), int(jj)])
sat_pix_mean = np.array(sat_pix_mean)
print("Mean position of saturated pixels blobs = \n", sat_pix_mean)
# if 0: # of scipy.ndimage
# df = ndimage.gaussian_filter(dataimage_ROI, 10)
# # histo = np.histogram(df)
# # print "histogram",histo
# # print "maxinten",np.amax(df)
# threshold_for_measurements = (
# np.amax(df) / 10.0
# ) # histo[1][1]# 1000 pour CdTe # 50 pour Ge
# tG = np.where(df > threshold_for_measurements, 1, 0)
# ll, nf = ndimage.label(tG) # , structure = np.ones((3,3)))
# meanpos = np.array(
# ndimage.measurements.center_of_mass(tG, ll, np.arange(1, nf + 1)), dtype=float)
# # meanpos = np.fliplr(meanpos) # this done later
# # print "meanpos",meanpos
# sat_pix_mean = meanpos
else:
print("No pixel saturation")
# SATURATION handling -(End) --------------------------------------------------------
# x,y from localmaxima is a matter of convention
peaki = peak[0] + imin
peakj = peak[1] + jmin
# building an array of hot pixels (2 coordinates)
Yarray = peakj
Xarray = peaki
peaklist = np.array([Xarray, Yarray]).T
# print peaklistfit2D
# print peaklist[100:150]
print("{} local maxima have been found".format(len(peaklist)))
# probing background and maximal intensity in boxsize
#
ptp_boxsize = boxsize_for_probing_minimal_value_background
tabptp = []
for k in list(range(len(peaklist))):
tabptp.append(minmax(dataimage_ROI, peaklist[k], ptp_boxsize, framedim=framedim))
ar_ptp = np.array(tabptp)
# ar_amp = np.subtract(ar_ptp[:,1],ar_ptp[:,0])
ar_amp = np.subtract(intensity_localmaxima, ar_ptp[:, 0])
amp_rank = np.argsort(ar_amp)[::-1]
peaklist_sorted = peaklist[amp_rank]
# ptp_sorted = ar_ptp[amp_rank]
amp_sorted = ar_amp[amp_rank]
# thresholding on peak-to-peak amplitude
threshold_amp = IntensityThreshold
cond = np.where(amp_sorted > threshold_amp)
th_peaklist = peaklist_sorted[cond]
th_ar_amp = amp_sorted[cond]
print("{} local maxima found after thresholding above {} amplitude above local background".format(
len(th_ar_amp), threshold_amp))
# remove duplicates (close points), the most intense pixel is kept
# minimum distance between hot pixel
# it corresponds both to distance between peaks and peak size ...
pixeldistance = pixeldistance_remove_duplicates
purged_pklist, index_todelete = GT.purgeClosePoints2(th_peaklist, pixeldistance)
purged_amp = np.delete(th_ar_amp, index_todelete)
print(
"{} local maxima found after removing duplicates (minimum intermaxima distance = {})".format(
len(purged_amp), pixeldistance))
# print "execution time : %f secondes"%( ttt.time() - time_0)
# merging different kind of peaks
if sat_pix_mean is not None:
print("Merging saturated and normal peaks")
print("number of saturated peaks : ", np.shape(sat_pix_mean)[0])
purged_pklist = np.vstack((sat_pix_mean, purged_pklist))
if 0: # check if there are still close hot pixels
disttable_c = ssd.pdist(purged_pklist, "euclidean")
maxdistance_c = np.amax(disttable_c)
sqdistmatrix_c = ssd.squareform(disttable_c)
distmatrix_c = sqdistmatrix_c + np.eye(sqdistmatrix_c.shape[0]) * maxdistance_c
# must be (array([], dtype=int64), array([], dtype=int64))
print("close hotpixels", np.where(distmatrix_c < pixeldistance))
# print "purged_pklist", purged_pklist
print("shape(purged_pklist)", np.shape(purged_pklist))
npeaks = np.shape(purged_pklist)[0]
Ipixmax = np.zeros(npeaks, dtype=int)
# print np.shape(Data)
for i in list(range(npeaks)):
Ipixmax[i] = Data[purged_pklist[i, 0], purged_pklist[i, 1]]
# print "Ipixmax = ", Ipixmax
return np.fliplr(purged_pklist), Ipixmax
[docs]def shiftarrays_accum(Data_array, n, dimensions=1, diags=0):
r"""
idem than shiftarrays() but with all intermediate shifted arrays
1D
returns 3 arrays corresponding to shifted arrays
by n in two directions and original one
2D
returns 5 arrays corresponding to shifted arrays
by n in two directions and original one
these arrays are ready for comparison with eg np.greater
Data_array must have shape (slowdim,fastdim) so that
slowdim-2*n>=1 and fastdim-2*n>=1
(ie central array with zero shift has some elements)
TODO: replace append by a pre allocated array
.. note:: readmccd.localmaxima is better
"""
if n <= 0:
raise ValueError("shift value must be positive")
if dimensions == 2:
if diags:
shift_zero = Data_array[n:-n, n:-n]
allleft = []
allright = []
allup = []
alldown = []
alldiagleftdown = [] # diag "y=x"
alldiagrightup = [] # diag "y=x"
alldiagrightdown = [] # diah "y=-x"
alldiagleftup = [] # diah "y=-x"
for k in np.arange(1, n + 1)[::-1]:
allleft.append(Data_array[n - k : -(n + k), n:-n])
alldown.append(Data_array[n:-n, n - k : -(n + k)])
alldiagrightdown.append(Data_array[n - k : -(n + k), n - k : -(n + k)])
if (n - k) != 0:
allright.append(Data_array[k + n : -(n - k), n:-n])
allup.append(Data_array[n:-n, k + n : -(n - k)])
alldiagleftdown.append(
Data_array[k + n : -(n - k), n - k : -(n + k)])
alldiagleftup.append(Data_array[k + n : -(n - k), k + n : -(n - k)])
alldiagrightup.append(
Data_array[n - k : -(n + k), k + n : -(n - k)])
else: # correct python array slicing at the end : a[n:0] would mean a[n:]
allright.append(Data_array[k + n :, n:-n])
allup.append(Data_array[n:-n, k + n :])
alldiagleftdown.append(Data_array[k + n :, n - k : -(n + k)])
alldiagleftup.append(Data_array[k + n :, k + n :])
alldiagrightup.append(Data_array[n - k : -(n + k), k + n :])
return (shift_zero,
allleft,
allright,
alldown,
allup,
alldiagleftdown,
alldiagrightup,
alldiagrightdown,
alldiagleftup)
else:
shift_zero = Data_array[n:-n, n:-n]
allleft = []
allright = []
allup = []
alldown = []
allleft.append(Data_array[: -2 * n, n:-n])
alldown.append(Data_array[n:-n, : -2 * n])
for k in np.arange(1, n)[::-1]:
allleft.append(Data_array[n - k : -(n + k), n:-n])
allright.append(Data_array[k + n : -(n - k), n:-n])
alldown.append(Data_array[n:-n, n - k : -(n + k)])
allup.append(Data_array[n:-n, k + n : -(n - k)])
allright.append(Data_array[2 * n :, n:-n])
allup.append(Data_array[n:-n, 2 * n :])
return shift_zero, allleft, allright, alldown, allup
elif dimensions == 1:
shift_zero = Data_array[n:-n]
allleft = []
allright = []
allleft.append(Data_array[: -2 * n])
for k in np.arange(1, n)[::-1]:
allright.append(Data_array[k + n : -(n - k)])
allleft.append(Data_array[n - k : -(n + k)])
allright.append(Data_array[2 * n :])
return shift_zero, allleft, allright
[docs]def LocalMaxima_from_thresholdarray(Data, IntensityThreshold=400, rois=None, framedim=None,
verbose=False):
r"""
return center of mass of each blobs composes by pixels above IntensityThreshold
if Centers = list of (x,y, halfboxsizex, halfboxsizey) perform only blob search in theses ROIs
.. warning:: center of mass of blob where all intensities are set to 1
"""
if rois is not None:
print('\n>>>>> Finding only peaks in %d ROIs.\n' % len(rois))
listmeanpos_roi = []
for x, y, boxx, boxy in rois:
centerj, centeri = x, y
boxj, boxi = boxx, boxy
(imin, imax, jmin, jmax) = (centeri - boxi, centeri + boxi + 1,
centerj - boxj, centerj + boxj + 1)
# avoid to wrong indices when slicing the data
imin, imax, jmin, jmax = check_array_indices(imin, imax, jmin, jmax,
framedim=framedim)
# print("imin, imax, jmin, jmax", imin, imax, jmin, jmax)
dataroi = Data[imin : imax, jmin : jmax]
if verbose:
print("\n------------------\nx,y, boxx, boxy", x, y, boxx, boxy)
print('max intensity in dataroi', np.amax(dataroi))
print('min intensity in dataroi', np.amin(dataroi))
print('IntensityThreshold', IntensityThreshold)
# other way equivalent
# print("framedim in LocalMaxima_from_thresholdarray", framedim)
# framedim = framedim[1], framedim[0]
# i1, i2, j1, j2 = getindices2cropArray((x,y), (boxx, boxy), framedim)
# # print "i1, i2, j1, j2-----", i1, i2, j1, j2
# dataroi = Data[i1:i2, j1:j2]
# # for spot near border, replace by zeros array
# if i2 - i1 != boxy * 2 or j2 - j1 != boxx * 2:
# dataroi = np.zeros((boxy * 2 + 1, boxx * 2 + 1))
# print('max intensity in dataroi 2 : ', np.amax(dataroi))
# blob seach in dataroi
thrData_for_label = np.where(dataroi > IntensityThreshold, 1, 0)
ll, nf = ndimage.label(thrData_for_label)
if nf == 0:
print('sad! No blobs there in this roi...')
continue
meanpos_roi = np.array(ndimage.measurements.maximum_position(dataroi, ll, np.arange(1, nf + 1)),
dtype=float)
if len(np.shape(meanpos_roi)) > 1:
meanpos_roi = np.fliplr(meanpos_roi)
else:
meanpos_roi = np.roll(meanpos_roi, 1)
if verbose:
print('meanpos_roi =>', meanpos_roi)
for pos in meanpos_roi:
listmeanpos_roi.append([pos[0] + x - boxx, pos[1] + y - boxy])
meanpos = np.array(listmeanpos_roi)
if verbose:
print('meanpos', meanpos)
# single ROI is whole Data
else:
thrData_for_label = np.where(Data > IntensityThreshold, 1, 0)
# thrData = np.where(Data > IntensityThreshold, Data, 0)
# star = array([[0,1,0],[1,1,1],[0,1,0]])
# ll, nf = ndimage.label(thrData_for_label, structure=np.ones((3,3)))
# ll, nf = ndimage.label(thrData_for_label, structure=star)
ll, nf = ndimage.label(thrData_for_label)
# print "nb of blobs in LocalMaxima_from_thresholdarray()", nf
if nf == 0:
return None
meanpos = np.array(ndimage.measurements.maximum_position(Data, ll, np.arange(1, nf + 1)),
dtype=float)
if len(np.shape(meanpos)) > 1:
meanpos = np.fliplr(meanpos)
else:
meanpos = np.roll(meanpos, 1)
return meanpos
[docs]def localmaxima(DataArray, n, diags=1, verbose=0):
r"""
from DataArray 2D returns (array([i1,i2,...,ip]),array([j1,j2,...,jp]))
of indices where pixels value is higher in two direction up to n pixels
this tuple can be easily used after in the following manner:
DataArray[tupleresult] is an array of the intensity of the hottest pixels in array
in similar way with only four cardinal directions neighbouring (found in the web):
import numpy as N
def local_minima(array2d):
return ((array2d <= np.roll(array2d, 1, 0)) &
(array2d <= np.roll(array2d, -1, 0)) &
(array2d <= np.roll(array2d, 1, 1)) &
(array2d <= np.roll(array2d, -1, 1)))
WARNING: flat top peak are not detected !!
"""
dim = len(np.shape(DataArray))
if diags:
c, alll, allr, alld, allu, diag11, diag12, diag21, diag22 = shiftarrays_accum(
DataArray, n, dimensions=dim, diags=diags)
flag = np.greater(c, alll[0])
for elem in alll[1:] + allr + alld + allu + diag11 + diag12 + diag21 + diag22:
flag = flag * np.greater(c, elem)
else:
c, alll, allr, alld, allu = shiftarrays_accum(DataArray, n, dimensions=dim, diags=diags)
flag = np.greater(c, alll[0])
for elem in alll[1:] + allr + alld + allu:
flag = flag * np.greater(c, elem)
peaklist = np.nonzero(flag) # in c frame index
if verbose:
print("value local max", c[peaklist])
print("value from original array ", DataArray[tuple(np.array(peaklist) + n)])
print("positions of local maxima in original frame index",
tuple(np.array(peaklist) + n),)
# first slow index array , then second fast index array
return tuple(np.array(peaklist) + n)
# --- ---- Filtering and background removal function
[docs]def gauss_kern(size, sizey=None):
r"""
Returns a normalized 2D gauss kernel array for convolutions
"""
size = int(size)
if sizey is not None:
sizey = size
else:
sizey = int(sizey)
x, y = np.mgrid[-1*size : size + 1, -1*sizey: sizey + 1]
g = np.exp(-(x ** 2 / float(size) + y ** 2 / float(sizey)))
return g / g.sum()
[docs]def blur_image(im, n, ny=None):
r"""
blurs the image by convolving with a gaussian kernel of typical
size n. The optional keyword argument ny allows for a different
size in the y direction.
"""
g = gauss_kern(n, sizey=ny)
improc = scipy.signal.convolve(im, g, mode="valid")
return improc
[docs]def blurCCD(im, n):
r"""
apply a blur filter to image ndarray
"""
framedim = im.shape
minipix = np.amin(im)
tab = minipix * np.ones(framedim)
blurredpart = blur_image(im, n)
tab[n:-n, n:-n] = blurredpart
return tab
[docs]def circularMask(center, radius, arrayshape):
r"""
return a boolean ndarray of elem in array inside a mask
"""
II, JJ = np.mgrid[0 : arrayshape[0], 0 : arrayshape[1]]
cond = (II - center[0]) ** 2 + (JJ - center[1]) ** 2 <= radius ** 2
return cond
[docs]def compute_autobackground_image(dataimage, boxsizefilter=10):
r"""
return 2D array of filtered data array
:param dataimage: array of image data
:type dataimage: 2D array
"""
bkgimage = filter_minimum(dataimage, boxsize=boxsizefilter)
return bkgimage
[docs]def computefilteredimage(dataimage, bkg_image, CCDlabel, kernelsize=5, formulaexpression="A-B",
usemask=True):
r"""
return 2D array of initial image data without background given by bkg_image data
usemask : True then substract bkg image on masked raw data
False apply formula on all pixels (no mask)
:param dataimage: array of image data
:type dataimage: 2D array
:param bkg_image: array of filtered image data (background)
:type bkg_image: 2D array
:param CCDlabel: key for CCD dictionary
:type CCDlabel: string
"""
framedim = DictLT.dict_CCD[CCDlabel][0]
SaturationLevel = DictLT.dict_CCD[CCDlabel][2]
dataformat = DictLT.dict_CCD[CCDlabel][5]
print("framedim in computefilteredimage ", framedim)
print("CCDlabel in computefilteredimage ", CCDlabel)
#
# if CCDlabel in ('EDF',):
# return dataimage
# if framedim not in ((2048, 2048), [2048, 2048]):
# raise ValueError, "Background removal still implemented for non squared camera "
# computing substraction on whole array
if CCDlabel in ("sCMOS", "sCMOS_fliplr"):
usemask = False
if usemask:
# mask parameter to avoid high intensity steps at border:
# TODO: to compute for all CCD types
# center, mask_radius, minvalue = (1024, 1024), 1010, 0
# radius of the mask a bit smaller than real radius avoiding circular intensity step
mask_radius = min(framedim[0] // 2, framedim[1] // 2) - 15
center, minvalue = (framedim[0] // 2, framedim[1] // 2), 0
print("mask_radius", mask_radius)
print("center", center)
dataarray2D_without_background = filterimage(dataimage, framedim,
blurredimage=bkg_image,
kernelsize=kernelsize,
mask_parameters=(center, mask_radius, minvalue),
clipvalues=(0, SaturationLevel),
imageformat=dataformat)
else:
dataarray2D_without_background = applyformula_on_images(dataimage, bkg_image,
formulaexpression=formulaexpression,
SaturationLevel=SaturationLevel,
clipintensities=True)
return dataarray2D_without_background
[docs]def filterimage(image_array, framedim, blurredimage=None,
kernelsize=5,
mask_parameters=None,
clipvalues=None,
imageformat=np.uint16):
r"""
compute a difference of images inside a region defined by a mask
:param blurredimage: ndarray image to substract to image_array
:param kernelsize: pixel size of gaussian kernel if blurredimage is None
:param mask_parameters: circular mask parameter: center=(x,y), radius, value outside mask
"""
if blurredimage is None:
dblur = blurCCD(image_array, kernelsize)
else:
dblur = blurredimage
if clipvalues is not None:
minival, maxival = clipvalues
tab = np.clip(image_array - dblur, minival, maxival)
else:
tab = image_array - dblur
if mask_parameters is not None:
center, radius, minvalue = mask_parameters
if minvalue == "minvalue":
minivalue = np.amin(image_array)
else:
minivalue = 0
maskcd = np.where(circularMask(center, radius, framedim), tab, minivalue)
return np.array(maskcd, dtype=imageformat)
[docs]def rebin2Darray(inputarray, bin_dims=(2, 2), operator="mean"):
r"""
rebin 2D array by applying an operator to define the value of one element from the other
:param operator: mean, min, max, sum
:param bin_dims: side sizes of binning. (2,3) means 2X3
"""
rows, cols = inputarray.shape
binr, binc = bin_dims
if operator == "mean":
op = np.mean
if operator == "max":
op = np.max
if operator == "min":
op = np.max
if operator == "sum":
op = np.sum
if rows % binr == 0 and cols % binc == 0:
return op(op(inputarray.reshape(rows // binr, binr, cols // binc, binc), axis=3), axis=1)
else:
print("array and binning size are not compatible")
return None
[docs]def blurCCD_with_binning(im, n, binsize=(2, 2)):
r"""
blur the array by rebinning before and after aplying the filter
"""
# framedim = im.shape
imrebin = rebin2Darray(im, bin_dims=binsize, operator="min")
if imrebin is None:
return None
dblur = blurCCD(imrebin, n)
return np.repeat(np.repeat(dblur, binsize[0], axis=0), binsize[1], axis=1)
[docs]def filter_minimum(im, boxsize=10):
r""" return filtered image using minimum filter"""
return ndimage.filters.minimum_filter(im, size=boxsize)
[docs]def remove_minimum_background(im, boxsize=10):
r"""
remove to image array the array resulting from minimum_filter
"""
return im - filter_minimum(im, boxsize=boxsize)
# --- -------------- Plot image and peaks
[docs]def plot_image_markers(image, markerpos, position_definition=1):
r"""
plot 2D array (image) with markers at first two columns of (markerpos)
.. note:: used in LaueHDF5. Could be better implementation in some notebooks
"""
fig = pp.figure()
ax = fig.add_subplot(111)
numrows, numcols = image.shape
def format_coord(x, y):
""" return string with x, y values """
col = int(x + 0.5)
row = int(y + 0.5)
if col >= 0 and col < numcols and row >= 0 and row < numrows:
z = image[row, col]
return "x = {:.4f}, y = {:.4f}, z = {:.4f}".format(x, y, z)
else:
return "x = {:.4f}, y = {:.4f}".format(x, y)
ax.format_coord = format_coord
from matplotlib.patches import Circle
# correction only to fitdata peak position to the display
if position_definition == 1:
PointToPlot = np.zeros(markerpos.shape)
PointToPlot[:, :2] = markerpos[:, :2] - np.array([1, 1])
for po in PointToPlot[:, :2]:
large_circle = Circle(po, 7, fill=False, color="b")
center_circle = Circle(po, 0.5, fill=True, color="r")
ax.add_patch(large_circle)
ax.add_patch(center_circle)
ax.imshow(np.log(image), interpolation="nearest")
pp.show()