[IPython-User] parallel computations on a global variable

Moritz Emanuel Beber moritz.beber@gmail....
Mon Feb 11 11:33:28 CST 2013


Dear all,

I'm trying to parallalize the computation of all pairwise inner products 
of the column vectors of a matrix. Unfortunately, I have some issues:

Since I want to avoid passing the large matrix many times, I pushed the 
matrix to the engines, making it available to the namespace there. I can 
access that matrix as a global just fine using a function defined in the 
notebook, doing the same in a module fails... (Please have a look at the 
two attached files.)

When I use an IPython.parallel.Reference to the global matrix and pass 
that to the function, that works in the external module. Of course, this 
means I have to pass a large list containing the reference to the map 
function.

Insights and advice is appreciated,
Moritz
-------------- next part --------------
A non-text attachment was scrubbed...
Name: ipy_parallel.py
Type: text/x-python
Size: 415 bytes
Desc: not available
Url : http://mail.scipy.org/pipermail/ipython-user/attachments/20130211/bb5535bc/attachment-0001.py 
-------------- next part --------------
{
 "metadata": {
  "name": "parallel_global_array"
 },
 "nbformat": 3,
 "nbformat_minor": 0,
 "worksheets": [
  {
   "cells": [
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "from IPython import sys_info"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 1
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "print sys_info()"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "{'commit_hash': 'a03b31a',\n",
        " 'commit_source': 'installation',\n",
        " 'default_encoding': 'UTF-8',\n",
        " 'ipython_path': '/usr/local/lib/python2.7/dist-packages/IPython',\n",
        " 'ipython_version': '0.14.dev',\n",
        " 'os_name': 'posix',\n",
        " 'platform': 'Linux-3.2.0-4-amd64-x86_64-with-debian-7.0',\n",
        " 'sys_executable': '/usr/bin/python',\n",
        " 'sys_platform': 'linux2',\n",
        " 'sys_version': '2.7.3 (default, Jan  2 2013, 13:56:14) \\n[GCC 4.7.2]'}\n"
       ]
      }
     ],
     "prompt_number": 2
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "import IPython.parallel as pp"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 3
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "rc = pp.Client()"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 4
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "rc.ids"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "output_type": "pyout",
       "prompt_number": 5,
       "text": [
        "[0, 1]"
       ]
      }
     ],
     "prompt_number": 5
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "dview = rc.direct_view()"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 6
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "a = numpy.random.random_sample((100, 100))"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 7
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "pp.require(\"numpy\")\n",
      "def parallel_inner(k, l):\n",
      "    return numpy.inner(a[:, k], a[:, l])"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 8
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "dview.push(dict(a=a))"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "output_type": "pyout",
       "prompt_number": 9,
       "text": [
        "<AsyncResult: _push>"
       ]
      }
     ],
     "prompt_number": 9
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "with dview.sync_imports():\n",
      "    import numpy"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "importing numpy on engine(s)\n"
       ]
      }
     ],
     "prompt_number": 10
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "result = dview.map(parallel_inner, [2, 3], [9, 15])"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 11
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "sum(result)"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "output_type": "pyout",
       "prompt_number": 12,
       "text": [
        "55.149183935285443"
       ]
      }
     ],
     "prompt_number": 12
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "import ipy_parallel"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 13
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "ipy_parallel.some_inner(dview, a)"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "ename": "RemoteError",
       "evalue": "NameError(global name 'mat' is not defined)",
       "output_type": "pyerr",
       "traceback": [
        "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
        "\u001b[1;31mNameError\u001b[0m                                 Traceback (most recent call last)\u001b[1;32m<string>\u001b[0m in \u001b[0;36m<module>\u001b[1;34m()\u001b[0m",
        "\u001b[1;32m/home/moritz/CodeBase/Development/biologis/applications/ipy_parallel.pyc\u001b[0m in \u001b[0;36mparallel_inner\u001b[1;34m(i, j)\u001b[0m",
        "\u001b[0;32m     15\u001b[0m \u001b[1;32mdef\u001b[0m \u001b[0mparallel_inner\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mi\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mj\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m",
        "\u001b[0;32m     16\u001b[0m     \u001b[1;31m# access to global variable `mat` on engine\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m",
        "\u001b[1;32m---> 17\u001b[1;33m     \u001b[1;32mreturn\u001b[0m \u001b[0mnumpy\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0minner\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mmat\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mi\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mmat\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mj\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m",
        "\u001b[0m\u001b[0;32m     18\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m",
        "\u001b[0;32m     19\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m",
        "\u001b[1;31mNameError\u001b[0m: global name 'mat' is not defined"
       ]
      }
     ],
     "prompt_number": 14
    }
   ],
   "metadata": {}
  }
 ]
}


More information about the IPython-User mailing list