[Numpy-discussion] RAM problem during code execution - Numpya arrays

Francesc Alted francesc@continuum...
Fri Aug 23 09:34:06 CDT 2013


Hi José,

The code is somewhat longish for a pure visual inspection, but my advice is
that you install memory profiler (
https://pypi.python.org/pypi/memory_profiler).  This will help you
determine which line or lines are hugging the memory the most.

Saludos,
Francesc

On Fri, Aug 23, 2013 at 3:58 PM, Josè Luis Mietta <
joseluismietta@yahoo.com.ar> wrote:

> Hi ecperts. I need your help with a RAM porblem during execution of my
> script.
> I wrote the next code. I use SAGE. In 1-2 hours of execution time the RAM
> of my laptop (8gb) is filled and the sistem crash:
>
> from scipy.stats import uniform    import numpy as np
>
> cant_de_cadenas =[700,800,900]
>
> cantidad_de_cadenas=np.array([])
> for kkkkk in cant_de_cadenas:
>     cantidad_de_cadenas=np.append(cantidad_de_cadenas,kkkkk)
>
> cantidad_de_cadenas=np.transpose(cantidad_de_cadenas)
>
> b=10
> h=bLongitud=1
> numero_experimentos=150
>
> densidad_de_cadenas =cantidad_de_cadenas/(b**2)
>
> prob_perc=np.array([])
>
> tiempos=np.array([])
>
> S_int=np.array([])
>
> S_medio=np.array([])
>
> desviacion_standard=np.array([])
>
> desviacion_standard_nuevo=np.array([])
>
> anisotropia_macroscopica_porcentual=np.array([])
>
> componente_y=np.array([])
>
> componente_x=np.array([])
> import time
> for N in cant_de_cadenas:
>
>     empieza=time.clock()
>
>     PERCOLACION=np.array([])
>
>     size_medio_intuitivo = np.array([])
>     size_medio_nuevo = np.array([])
>     std_dev_size_medio_intuitivo = np.array([])
>     std_dev_size_medio_nuevo = np.array([])
>     comp_y = np.array([])
>     comp_x = np.array([])
>
>
>     for u in xrange(numero_experimentos):
>
>
>         perco = False
>
>         array_x1=uniform.rvs(loc=-b/2, scale=b, size=N)
>         array_y1=uniform.rvs(loc=-h/2, scale=h, size=N)
>         array_angle=uniform.rvs(loc=-0.5*(np.pi), scale=np.pi, size=N)
>
>
>         array_pendiente_x=1./np.tan(array_angle)
>
>         random=uniform.rvs(loc=-1, scale=2, size=N)
>         lambda_sign=np.zeros([N])
>         for t in xrange(N):
>             if random[t]<0:
>                 lambda_sign[t]=-1
>             else:
>                 lambda_sign[t]=1
>         array_lambdas=(lambda_sign*Longitud)/np.sqrt(1+array_pendiente_x**2)
>
>
>         array_x2= array_x1 + array_lambdas*array_pendiente_x
>         array_y2= array_y1 + array_lambdas*1
>
>         array_x1 = np.append(array_x1, [-b/2, b/2, -b/2, -b/2])
>         array_y1 = np.append(array_y1, [-h/2, -h/2, -h/2, h/2])
>         array_x2 = np.append(array_x2, [-b/2, b/2, b/2, b/2])
>         array_y2 = np.append(array_y2, [h/2, h/2, -h/2, h/2])
>
>         M = np.zeros([N+4,N+4])
>
>         for j in xrange(N+4):
>             if j>0:
>                 x_A1B1 = array_x2[j]-array_x1[j]
>                 y_A1B1 = array_y2[j]-array_y1[j]
>                 x_A1A2 = array_x1[0:j]-array_x1[j]
>                 y_A1A2 = array_y1[0:j]-array_y1[j]
>                 x_A2A1 = -1*x_A1A2
>                 y_A2A1 = -1*y_A1A2
>                 x_A2B2 = array_x2[0:j]-array_x1[0:j]
>                 y_A2B2 = array_y2[0:j]-array_y1[0:j]
>                 x_A1B2 = array_x2[0:j]-array_x1[j]
>                 y_A1B2 = array_y2[0:j]-array_y1[j]
>                 x_A2B1 = array_x2[j]-array_x1[0:j]
>                 y_A2B1 = array_y2[j]-array_y1[0:j]
>
>                 p1 = x_A1B1*y_A1A2 - y_A1B1*x_A1A2
>                 p2 = x_A1B1*y_A1B2 - y_A1B1*x_A1B2
>                 p3 = x_A2B2*y_A2B1 - y_A2B2*x_A2B1
>                 p4 = x_A2B2*y_A2A1 - y_A2B2*x_A2A1
>
>                 condicion_1=p1*p2
>                 condicion_2=p3*p4
>
>                 for k in xrange (j):
>                     if condicion_1[k]<=0 and condicion_2[k]<=0:
>                         M[j,k]=1
>                 del condicion_1
>                 del condicion_2
>
>
>             if j+1<N+4:
>                 x_A1B1 = array_x2[j]-array_x1[j]
>                 y_A1B1 = array_y2[j]-array_y1[j]
>                 x_A1A2 = array_x1[j+1:]-array_x1[j]
>                 y_A1A2 = array_y1[j+1:]-array_y1[j]
>                 x_A2A1 = -1*x_A1A2
>                 y_A2A1 = -1*y_A1A2
>                 x_A2B2 = array_x2[j+1:]-array_x1[j+1:]
>                 y_A2B2 = array_y2[j+1:]-array_y1[j+1:]
>                 x_A1B2 = array_x2[j+1:]-array_x1[j]
>                 y_A1B2 = array_y2[j+1:]-array_y1[j]
>                 x_A2B1 = array_x2[j]-array_x1[j+1:]
>                 y_A2B1 = array_y2[j]-array_y1[j+1:]
>
>                 p1 = x_A1B1*y_A1A2 - y_A1B1*x_A1A2
>                 p2 = x_A1B1*y_A1B2 - y_A1B1*x_A1B2
>                 p3 = x_A2B2*y_A2B1 - y_A2B2*x_A2B1
>                 p4 = x_A2B2*y_A2A1 - y_A2B2*x_A2A1
>
>                 condicion_1=p1*p2
>                 condicion_2=p3*p4
>
>                 for k in xrange ((N+4)-j-1):
>                     if condicion_1[k]<=0 and condicion_2[k]<=0:
>                         M[j,k+j+1]=1
>                 del condicion_1
>                 del condicion_2
>
>         M[N,N+2]=0
>         M[N,N+3]=0
>         M[N+1,N+2]=0
>         M[N+1,N+3]=0
>         M[N+2,N]=0
>         M[N+2,N+1]=0
>         M[N+3,N]=0
>         M[N+3,N+1]=0
>
>
>         CD=np.array([])
>
>         POPOPO=[]
>         for g in xrange(N):
>
>             lala=0
>             r=False
>             while lala<=len(POPOPO)-1:
>                 esta= g in POPOPO[lala]
>
>                 if esta is True:
>                     lala=len(POPOPO)
>                     r=True
>                 else:
>                     lala=lala+1
>             if r is False:
>
>
>
>                 L=np.array([g])
>                 for s in xrange(N):
>                     if M[g,s] != 0:
>                         L=np.append(L,s)
>
>                 x=0
>                 while x<= N:
>                     for l in xrange(N):
>                         z= l in L
>                         d=L[x]
>                         if z is False and M[d,l] != 0:
>                             L=np.append(L,l)
>                     if x+1<len(L):
>                         x+=1
>                     else:
>                         x=N+1.
>
>                 q= len (L)
>                 CD=np.append(CD, q)
>                 POPOPO.append(L)
>
>
>         M_horizontal=M.copy()
>         M_horizontal[:,N+2] = np.zeros(N+4)
>         M_horizontal[:,N+3] = np.zeros(N+4)
>         M_horizontal[N+2] = np.zeros(N+4)
>         M_horizontal[N+3] = np.zeros(N+4)
>
>
>
>         L=np.array([N])
>         for s in xrange(N+4):
>             if M_horizontal[N,s] != 0:
>                 L=np.append(L,s)
>
>         x=0
>         while x<= N+4:
>             for l in xrange(N+4):
>                 z= l in L
>                 d=L[x]
>                 if z is False and M_horizontal[d,l] != 0:
>                     L=np.append(L,l)
>             if x+1<len(L):
>                 x+=1
>             else:
>                 x=(N+4)+1.
>
>         LV1_in_L = N in L
>         LV2_in_L= (N+1) in L
>
>         if LV1_in_L is True and LV2_in_L is True:
>             perc_horiz=True
>         else:
>             perc_horiz=False
>
>
>         M_vertical=M.copy()
>
>         M_vertical[:,N] = np.zeros(N+4)
>         M_vertical[:,N+1] = np.zeros(N+4)
>         M_vertical[N] = np.zeros(N+4)
>         M_vertical[N+1] = np.zeros(N+4)
>
>
>
>         L=np.array([N+2])
>         for s in xrange(N+4):
>             if M_vertical[N+2,s] != 0:
>                 L=np.append(L,s)
>
>         x=0
>         while x<= N+4:
>             for l in xrange(N+4):
>                 z= l in L
>                 d=L[x]
>                 if z is False and M_vertical[d,l] != 0:
>                     L=np.append(L,l)
>             if x+1<len(L):
>                 x+=1
>             else:
>                 x=(N+4)+1.
>
>         LH1_in_L = (N+2) in L
>         LH2_in_L= (N+3) in L
>
>         if LH1_in_L is True and LH2_in_L is True:
>             perc_ver = True
>         else:
>             perc_ver = False
>
>
>
>         if perc_ver is True or perc_horiz is True:
>             PERCOLACION=np.append(PERCOLACION,1)
>             perco=True
>
>
>         D = np.array([])
>         W = np.array([])
>
>         for c in xrange (int(min(CD)), int(max(CD)+1),1):
>             D=np.append(D,c)
>             frec = sum (CD == c)
>             W = np.append(W,frec)
>
>         if perco is True:
>             posicion=np.argmax(D)
>             D=np.delete(D,posicion)
>             W=np.delete(W,posicion)
>
>         if len(D) == 0 and len(W)==0:
>             S_medio_intuitivo_exp_u=0
>             S_medio_nuevo_exp_u = 0
>             std_dev_exp_u = 0
>             std_dev_nuevo_exp_u = 0
>         else:
>
>             S_medio_intuitivo_exp_u = np.average (D,weights=W)
>
>             peso_nuevo=D*W
>
>             S_medio_nuevo_exp_u = np.average (D,weights=peso_nuevo)
>
>
>             tipos=sum(W)
>             X=W*((D-S_medio_intuitivo_exp_u)**2)
>             S=sum(X)
>
>             std_dev_exp_u = np.sqrt(S/(tipos-1.))
>
>             tipos_nuevo=sum(peso_nuevo)
>             X_nuevo=peso_nuevo*((D-S_medio_nuevo_exp_u)**2)
>             S_nuevo=sum(X_nuevo)
>
>             std_dev_nuevo_exp_u = np.sqrt(S_nuevo/(tipos_nuevo-1.))
>
>
>         componente_longitudinal=Longitud*np.abs(np.cos(array_angle))
>         comp_y=np.append(comp_y, sum(componente_longitudinal)/N)
>
>         componente_transversal=Longitud*np.abs(np.sin(array_angle))
>         comp_x=np.append(comp_x, sum(componente_transversal)/N)
>
>         std_dev_size_medio_intuitivo=np.append(std_dev_size_medio_intuitivo, std_dev_exp_u)
>
>         std_dev_size_medio_nuevo=np.append(std_dev_size_medio_nuevo, std_dev_nuevo_exp_u)
>
>         size_medio_intuitivo=np.append(size_medio_intuitivo, S_medio_intuitivo_exp_u)
>
>         size_medio_nuevo=np.append(size_medio_nuevo, S_medio_nuevo_exp_u)
>
>
>     percolation_probability=sum(PERCOLACION)/numero_experimentos
>
>     prob_perc=np.append(prob_perc, percolation_probability)
>
>     S_int = np.append (S_int, sum(size_medio_intuitivo)/numero_experimentos)
>
>     S_medio=np.append (S_medio, sum(size_medio_nuevo)/numero_experimentos)
>
>     desviacion_standard = np.append (desviacion_standard, sum(std_dev_size_medio_intuitivo)/numero_experimentos)
>
>     desviacion_standard_nuevo=np.append (desviacion_standard_nuevo, sum(std_dev_size_medio_nuevo)/numero_experimentos)
>
>     tiempos=np.append(tiempos, time.clock()-empieza)
>
>     componente_y=np.append(componente_y, sum(comp_y)/numero_experimentos)
>     componente_x=np.append(componente_x, sum(comp_x)/numero_experimentos)
>
>     anisotropia_macroscopica_porcentual=100*(1-(componente_y/componente_x))
>
> I tryed with gc and gc.collect() and 'del'command for deleting arrays
> after his use and nothing work!
>
> What am I doing wrong? Why the memory becomes full while running (starts
> with 10% of RAM used and in 1-2hour is totally full used)?
>
> Please help me, I'm totally stuck!
> Thanks a lot!
>
> _______________________________________________
> NumPy-Discussion mailing list
> NumPy-Discussion@scipy.org
> http://mail.scipy.org/mailman/listinfo/numpy-discussion
>
>


-- 
Francesc Alted
-------------- next part --------------
An HTML attachment was scrubbed...
URL: http://mail.scipy.org/pipermail/numpy-discussion/attachments/20130823/63c94c96/attachment-0001.html 


More information about the NumPy-Discussion mailing list