2015-04-30 trame-sensorielle

L'installation Elasticité dynamique agit comme un filtre et génère de nouveaux espaces démultipliés, comme un empilement quasi infini d'horizons. Par principe de réflexion, la pièce absorbe l'image de l'environnement et accumule les points de vue ; le mouvement permanent requalifie continuellement ce qui est regardé et entendu.

Ce post utilise une image naturelle comme entrée d'une "trame sensorielle".

At each time, the pipeline is the following:

  • take an image,
  • turn into blocks corresponding to the edges' centers,
  • into each block determine the most likely orientation

Let's first create a dummy movie:

In [37]:
%load_ext autoreload
%autoreload 2
The autoreload extension is already loaded. To reload it, use:
  %reload_ext autoreload
In [38]:
import os
import matplotlib
matplotlib.use("Agg") # agg-backend, so we can create figures without x-server (no PDF, just PNG etc.)
from elasticite import EdgeGrid
e = EdgeGrid()
fps = 24.
loop = 1
autoplay = 0
duration = 4.
figpath = '../files/elasticite/'
In [39]:
import matplotlib as mpl
import matplotlib.pyplot as plt
from moviepy.video.io.bindings import mplfig_to_npimage
import moviepy.editor as mpy

fig_mpl, ax = plt.subplots(1, figsize=(4, 4), facecolor='white')

def draw_elementary_pattern(ax, center): 
    #ax.add_artist(mpl.patches.Wedge(center-1., 1., 0, 180, width=.1)) #center, r, theta1, theta2, width=None
    #ax.add_artist(mpl.patches.Wedge(-center+1., 1., 0, 180, width=.1))
    # class matplotlib.patches.RegularPolygon(xy, numVertices, radius=5, orientation=0, **kwargs)¶
    ax.add_artist(mpl.patches.RegularPolygon((.5,.5), 5, center, facecolor='r'))
    ax.add_artist(mpl.patches.RegularPolygon((.5,.5), 5, center/2, facecolor='w'))
    ax.add_artist(mpl.patches.RegularPolygon((.5,.5), 5, center/4, facecolor='g'))

def make_frame_mpl(t):
    ax = fig_mpl.add_axes([0., 0., 1., 1.], axisbg='w')
    ax.cla()
    plt.setp(ax, xticks=[])
    plt.setp(ax, yticks=[])
    #ax.axis(c='b', lw=0, frame_on=False)
    ax.grid(b=False, which="both")
    draw_elementary_pattern(ax, t/duration)
    return mplfig_to_npimage(fig_mpl) # RGB image of the figure

animation = mpy.VideoClip(make_frame_mpl, duration=duration)
In [40]:
name, vext = 'elasticite_test', '.mp4'
fname = os.path.join(figpath, name + vext)
if not os.path.isfile(fname):
    animation.write_videofile(fname, fps=fps)
mpy.ipython_display(fname)
Out[40]:

Now read this clip using imageio:

In [41]:
import imageio

reader = imageio.get_reader(fname)
for i, im in enumerate(reader):
    print('Mean of frame %i is %1.1f' % (i, im.mean()))
    if i > 15: break
Mean of frame 0 is 247.5
Mean of frame 1 is 247.5
Mean of frame 2 is 247.2
Mean of frame 3 is 246.8
Mean of frame 4 is 246.2
Mean of frame 5 is 245.6
Mean of frame 6 is 244.9
Mean of frame 7 is 244.2
Mean of frame 8 is 243.4
Mean of frame 9 is 242.5
Mean of frame 10 is 241.6
Mean of frame 11 is 240.6
Mean of frame 12 is 239.4
Mean of frame 13 is 238.4
Mean of frame 14 is 237.1
Mean of frame 15 is 235.7
Mean of frame 16 is 234.3

Let's consider one frame, and a ROI defined by its center and width:

In [42]:
def mat2ipn(mat):   
    # create a temporary file
    import tempfile
    filename = tempfile.mktemp(suffix='.png')
    # Use write_png to export your wonderful plot as png ! 
    #import vispy.io as io
    imageio.imwrite(filename, mat)
    from IPython.core.display import display, Image
    return display(Image(filename))    
mat2ipn(im)
#from holoviews import Image
#from holoviews import HoloMap, Dimension
#%load_ext holoviews.ipython

trying to guess orientations using LogGabors

In [43]:
from NeuroTools.parameters import ParameterSet
from SLIP import Image
import numpy as np
print(type(im))
slip = Image(np.array(im))
<class 'imageio.core.util.Image'>
In [44]:
im_ = im.sum(axis=-1)
print(im_.shape)

X_=.3
Y_=.5
w=.2
im_ = im_ * np.exp(-.5*((slip.x-X_)**2+(slip.y-Y_)**2)/w**2)
mat2ipn(im_)
(80, 80)

Now, we will test the energy of different orientations :

In [45]:
from LogGabor import LogGabor
lg = LogGabor(np.array(im))
N_theta = 24
thetas, E = np.linspace(0, np.pi, N_theta), np.zeros((N_theta,))
for i_theta, theta in enumerate(thetas):
    params= {'sf_0':.3, 'B_sf': .3, 'theta':theta, 'B_theta': .1}
    FT_lg = lg.loggabor(0, 0, **params)
    E[i_theta] = np.sum(np.absolute(slip.FTfilter(im_.T, FT_lg, full=True))**2)
    print(theta*180/np.pi, E[i_theta])       
0.0 422966.590084
7.82608695652 1669343.25201
15.652173913 4552116.25284
23.4782608696 3625218.0053
31.3043478261 1003327.12766
39.1304347826 301222.976478
46.9565217391 178297.502749
54.7826086957 150824.70314
62.6086956522 174399.412664
70.4347826087 244104.969851
78.2608695652 395425.138414
86.0869565217 572116.531448
93.9130434783 531507.589971
101.739130435 269689.812316
109.565217391 83851.3638405
117.391304348 56114.5738155
125.217391304 67487.3655702
133.043478261 70332.9778141
140.869565217 59227.2662327
148.695652174 45316.972146
156.52173913 37393.8670279
164.347826087 62625.6380967
172.173913043 142226.690291
180.0 422966.590084

Now select the most likely:

In [46]:
print(np.argmax(E), thetas[np.argmax(E)]*180/np.pi)
2 15.652173913

wrapping things in one function:

In [47]:
from NeuroTools.parameters import ParameterSet
from SLIP import Image
from LogGabor import LogGabor
import numpy as np
lg = LogGabor(np.array(im))
N_theta = 24
def theta_max(im, X_=.0, Y_=.0, w=.3):
    im_ = im.sum(axis=-1)
    im_ = im_ * np.exp(-.5*((slip.x-X_)**2+(slip.y-Y_)**2)/w**2)
    thetas, E = np.linspace(0, np.pi, N_theta), np.zeros((N_theta,))
    for i_theta, theta in enumerate(thetas):
        params= {'sf_0':.3, 'B_sf': .3, 'theta':theta, 'B_theta': .1}
        FT_lg = lg.loggabor(0, 0, **params)
        E[i_theta] = np.sum(np.absolute(slip.FTfilter(im_.T, FT_lg, full=True))**2)
    return np.pi/2 - thetas[np.argmax(E)]     

e.reader = imageio.get_reader(fname, loop=True)
for i, im in enumerate(reader):
    print(i, theta_max(im, X_=.3, Y_=.3, w=.3)*180./np.pi)
    if i > 5: break
0 90.0
1 -90.0
2 -90.0
3 90.0
4 90.0
5 -3.91304347826
6 74.347826087

We retrieve the centers and span of all edges from the EdgeGrid class:

name = 'trame_loggabor' import numpy as np from elasticite import EdgeGrid e = EdgeGrid() import imageio e.reader = imageio.get_reader(fname, loop=True) def make_lames(e): im = e.reader.get_next_data() for i in range(e.N_lame): e.lames[2, i] = e.theta_max(im, X_=e.lames[0, i], Y_=e.lames[1, i], w=.05) return e.lames[2, :] e.make_anim(name, make_lames, duration=duration) e.ipython_display(name)

trying to guess orientations using Sobel filters

dans ce cas, on voit que les filtres orientés sont corrects, mais c'est un peu overkill (et lent) donc on peut préférer utiliser des filtres orientés plus simples, les filtres de Sobel, soit pour les horizontales la matrice:

[1   2  1]
[0   0  0]
[-1 -2 -1]    

et son transposé (pour les verticales).

In [48]:
name = 'trame_sobel_orientations'
if not os.path.isfile(os.path.join(figpath, name + '.webm')):
    e = EdgeGrid()

    import imageio
    e.reader = imageio.get_reader(fname, loop=True)

    import matplotlib.pyplot as plt
    import numpy as np
    from moviepy.video.io.bindings import mplfig_to_npimage
    import moviepy.editor as mpy

    # DRAW A FIGURE WITH MATPLOTLIB
    fps = 24.
    duration = 4.
    fig_mpl, ax = plt.subplots(1, 2, figsize=(10,5), facecolor='white')
    def make_frame_mpl(t):
        import numpy as np
        sobel = np.array([[1,   2,  1,],
                          [0,   0,  0,],
                          [-1, -2, -1,]])
        im = e.reader.get_next_data()

        im_ = im.sum(axis=-1)
        from scipy.signal import convolve2d
        #im_ = im_ * np.exp(-.5*((slip.x-X_)**2+(slip.y-Y_)**2)/w**2)
        ax[0].imshow(convolve2d(im_, sobel, 'same'))
        ax[1].imshow(convolve2d(im_, sobel.T, 'same'))
        return mplfig_to_npimage(fig_mpl) # RGB image of the figure

    animation = mpy.VideoClip(make_frame_mpl, duration=duration)
    animation.write_videofile(os.path.join(figpath, name + '.webm'), fps=fps)
e.ipython_display(name)

The angle is derived as the arctan of the 2 components

In [49]:
name = 'trame_sobel_orientation'
import os
if True or not os.path.isfile(os.path.join(figpath, name + '.webm')):
    e = EdgeGrid()

    import imageio
    e.reader = imageio.get_reader(fname, loop=True)

    import matplotlib.pyplot as plt
    import numpy as np
    from moviepy.video.io.bindings import mplfig_to_npimage
    import moviepy.editor as mpy

    # DRAW A FIGURE WITH MATPLOTLIB
    fps = 24.
    duration = 4.
    
    
    def make_frame_mpl(t):
        fig_mpl, ax = plt.subplots(figsize=(5,5), facecolor='white')

        import numpy as np
        sobel = np.array([[1,   2,  1],
                          [0,   0,  0],
                          [-1, -2, -1]])
        im = e.reader.get_next_data()

        im_ = im.sum(axis=-1)
        N_X, N_Y = im_.shape
        x, y = np.mgrid[0:1:1j*N_X, 0:1:1j*N_Y]
        # mask = np.exp(-.5*((x-.5)**2+(y-.5)**2)/.1**2)       
        blur = np.array([[1, 2, 1],
                         [1, 8, 2],
                         [1, 2, 1]])

        from scipy.signal import convolve2d
        im_X = convolve2d(im_, sobel, 'same')
        im_Y = convolve2d(im_, sobel.T, 'same')
        for i in range(10):
            im_X = convolve2d(im_X, blur, 'same')
            im_Y = convolve2d(im_Y, blur, 'same')
        mappable = ax.imshow(np.arctan2(im_Y, im_X)*180/np.pi, origin='lower')
        fig_mpl.colorbar(mappable)
        return mplfig_to_npimage(fig_mpl) # RGB image of the figure

    animation = mpy.VideoClip(make_frame_mpl, duration=duration)
    animation.write_videofile(os.path.join(figpath, name + '.webm'), fps=fps)
e.ipython_display(name)
[MoviePy] >>>> Building video ../files/elasticite/trame_sobel_orientation.webm
[MoviePy] Writing video ../files/elasticite/trame_sobel_orientation.webm
                                               
[MoviePy] Done.
[MoviePy] >>>> Video ready: ../files/elasticite/trame_sobel_orientation.webm 


This function is included in the EdgeGrid class:

In [ ]:
e = EdgeGrid()
import numpy as np
np.set_printoptions(precision=2, suppress=True)
import imageio
e.reader = imageio.get_reader(fname, loop=True)
for i, im in enumerate(e.reader):
    print(i, e.theta_sobel(im, N_blur=10)*180/np.pi)
    if i>5: break
In [ ]:
name = 'trame_sobel'
e = EdgeGrid()

import imageio
e.reader = imageio.get_reader(fname, loop=True)
def make_lames(e):
    e.im = e.reader.get_next_data()
    return e.theta_sobel(e.im, N_blur=10)

duration = 4.
e.make_anim(name, make_lames, duration=duration)
e.ipython_display(name)
In [57]:
import imagen as ig
line=ig.Line(xdensity=5, ydensity=5, smoothing=0)
import numpy as np
np.set_printoptions(1)
import holoviews
%reload_ext holoviews.ipython
In [58]:
import numbergen as ng
from holoviews import NdLayout
import param
param.Dynamic.time_dependent=True
stim = ig.SineGrating(orientation=np.pi*ng.UniformRandom())
NdLayout(stim.anim(3))
Out[58]:
In [ ]:
name = 'trame_sobel_grating'
e = EdgeGrid()
stim = ig.SineGrating(xdensity=64, ydensity=64)
def make_lames(e):
    stim.orientation=np.pi*e.t/4.
    e.im = stim()
    return e.theta_sobel(e.im, N_blur=5)

duration = 4.
e.make_anim(name, make_lames, duration=duration)
e.ipython_display(name)
In [ ]:
%%opts Image.Pattern (cmap='Blues_r')
l1 = ig.Line(orientation=-np.pi/4)
l2 = ig.Line(orientation=+np.pi/4)
cross = l1 | l2
cross.orientation=ng.ScaledTime()*(np.pi/-20)
l1.anim(20) + l2.anim(20) + cross.anim(20)
In [ ]:
name = 'trame_sobel_cross'
e = EdgeGrid()
l1 = ig.Line(orientation=-np.pi/4)
l2 = ig.Line(orientation=+np.pi/4)
cross = l1 | l2

def make_lames(e):
    cross.orientation = np.pi*e.t/4.
    e.im = cross()
    return e.theta_sobel(e.im, N_blur=1)

duration = 4.
e.make_anim(name, make_lames, duration=duration)
e.ipython_display(name)
In [60]:
line.set_param(xdensity=72,ydensity=72,orientation=np.pi/4, thickness=0.02, smoothing=0.02)
line.x = .25

noise = ig.Composite(xdensity=72, ydensity=72,
                     operator=np.add,
                     generators=[ig.Gaussian(size=0.1,
                                          x=ng.UniformRandom(seed=i+1)-0.5,
                                          y=ng.UniformRandom(seed=i+2)-0.5,
                                          orientation=np.pi*ng.UniformRandom(seed=i+3))
                                for i in range(10)])

stim = line + 0.3*noise
NdLayout(stim.anim(4)).cols(5)
Out[60]:
In [61]:
name = 'trame_sobel_line_tmp_4'
e = EdgeGrid()

def make_lames(e):
    line.x = -.5 + e.t / 4.
    stim = line + noise
    e.im = stim()
    return e.theta_sobel(e.im, N_blur=1)

duration = 4.
e.make_anim(name, make_lames, duration=duration)
e.ipython_display(name)
---------------------------------------------------------------------------
IndexError                                Traceback (most recent call last)
<ipython-input-61-fd5def21fc70> in <module>()
      9 
     10 duration = 4.
---> 11 e.make_anim(name, make_lames, duration=duration)
     12 e.ipython_display(name)

/Users/lolo/cloud_nas/science/2016-01-13_elasticite_github/src/elasticite.py in make_anim(self, name, make_lames, duration, redo)
    538                 return mplfig_to_npimage(fig_mpl) #  RGB image of the figure
    539 
--> 540             animation = mpy.VideoClip(make_frame_mpl, duration=duration)
    541             animation.write_videofile(self.fname(name), fps=self.fps)
    542 

/usr/local/lib/python3.5/site-packages/moviepy/video/VideoClip.py in __init__(self, make_frame, ismask, duration, has_constant_size)
    104         if make_frame is not None:
    105             self.make_frame = make_frame
--> 106             self.size = self.get_frame(0).shape[:2][::-1]
    107         self.ismask = ismask
    108         self.has_constant_size=has_constant_size

/usr/local/lib/python3.5/site-packages/moviepy/Clip.py in get_frame(self, t)

/usr/local/lib/python3.5/site-packages/moviepy/decorators.py in wrapper(f, *a, **kw)
     87         new_kw = {k: fun(v) if k in varnames else v
     88                  for (k,v) in kw.items()}
---> 89         return f(*new_a, **new_kw)
     90     return decorator.decorator(wrapper)
     91 

/usr/local/lib/python3.5/site-packages/moviepy/Clip.py in get_frame(self, t)
     93                 return frame
     94         else:
---> 95             return self.make_frame(t)
     96 
     97     def fl(self, fun, apply_to=[] , keep_duration=True):

/Users/lolo/cloud_nas/science/2016-01-13_elasticite_github/src/elasticite.py in make_frame_mpl(t)
    532 #                  on ne peut changer que l'orientation des lames:
    533                 self.t = t
--> 534                 self.lames[2, :] = make_lames(self)
    535                 self.update_lines()
    536                 fig_mpl, ax = self.show_edges()

<ipython-input-61-fd5def21fc70> in make_lames(e)
      6     stim = line + noise
      7     e.im = stim()
----> 8     return e.theta_sobel(e.im, N_blur=1)
      9 
     10 duration = 4.

/Users/lolo/cloud_nas/science/2016-01-13_elasticite_github/src/elasticite.py in theta_sobel(self, im, N_blur, w)
    289         for i in range(self.N_lame):
    290             angles[i] = angle[int((bord+self.lames[0, i]*(1-2*bord))*N_X),
--> 291                               int((bord+self.lames[1, i]*(1-2*bord))*N_Y)]
    292         return angles - np.pi/2
    293 

IndexError: index -759 is out of bounds for axis 0 with size 256

Comments

Comments powered by Disqus