[IPython-User] parallel computations on a global variable
Moritz Emanuel Beber
moritz.beber@gmail....
Mon Feb 11 11:33:28 CST 2013
Dear all,
I'm trying to parallalize the computation of all pairwise inner products
of the column vectors of a matrix. Unfortunately, I have some issues:
Since I want to avoid passing the large matrix many times, I pushed the
matrix to the engines, making it available to the namespace there. I can
access that matrix as a global just fine using a function defined in the
notebook, doing the same in a module fails... (Please have a look at the
two attached files.)
When I use an IPython.parallel.Reference to the global matrix and pass
that to the function, that works in the external module. Of course, this
means I have to pass a large list containing the reference to the map
function.
Insights and advice is appreciated,
Moritz
-------------- next part --------------
A non-text attachment was scrubbed...
Name: ipy_parallel.py
Type: text/x-python
Size: 415 bytes
Desc: not available
Url : http://mail.scipy.org/pipermail/ipython-user/attachments/20130211/bb5535bc/attachment-0001.py
-------------- next part --------------
{
"metadata": {
"name": "parallel_global_array"
},
"nbformat": 3,
"nbformat_minor": 0,
"worksheets": [
{
"cells": [
{
"cell_type": "code",
"collapsed": false,
"input": [
"from IPython import sys_info"
],
"language": "python",
"metadata": {},
"outputs": [],
"prompt_number": 1
},
{
"cell_type": "code",
"collapsed": false,
"input": [
"print sys_info()"
],
"language": "python",
"metadata": {},
"outputs": [
{
"output_type": "stream",
"stream": "stdout",
"text": [
"{'commit_hash': 'a03b31a',\n",
" 'commit_source': 'installation',\n",
" 'default_encoding': 'UTF-8',\n",
" 'ipython_path': '/usr/local/lib/python2.7/dist-packages/IPython',\n",
" 'ipython_version': '0.14.dev',\n",
" 'os_name': 'posix',\n",
" 'platform': 'Linux-3.2.0-4-amd64-x86_64-with-debian-7.0',\n",
" 'sys_executable': '/usr/bin/python',\n",
" 'sys_platform': 'linux2',\n",
" 'sys_version': '2.7.3 (default, Jan 2 2013, 13:56:14) \\n[GCC 4.7.2]'}\n"
]
}
],
"prompt_number": 2
},
{
"cell_type": "code",
"collapsed": false,
"input": [
"import IPython.parallel as pp"
],
"language": "python",
"metadata": {},
"outputs": [],
"prompt_number": 3
},
{
"cell_type": "code",
"collapsed": false,
"input": [
"rc = pp.Client()"
],
"language": "python",
"metadata": {},
"outputs": [],
"prompt_number": 4
},
{
"cell_type": "code",
"collapsed": false,
"input": [
"rc.ids"
],
"language": "python",
"metadata": {},
"outputs": [
{
"output_type": "pyout",
"prompt_number": 5,
"text": [
"[0, 1]"
]
}
],
"prompt_number": 5
},
{
"cell_type": "code",
"collapsed": false,
"input": [
"dview = rc.direct_view()"
],
"language": "python",
"metadata": {},
"outputs": [],
"prompt_number": 6
},
{
"cell_type": "code",
"collapsed": false,
"input": [
"a = numpy.random.random_sample((100, 100))"
],
"language": "python",
"metadata": {},
"outputs": [],
"prompt_number": 7
},
{
"cell_type": "code",
"collapsed": false,
"input": [
"pp.require(\"numpy\")\n",
"def parallel_inner(k, l):\n",
" return numpy.inner(a[:, k], a[:, l])"
],
"language": "python",
"metadata": {},
"outputs": [],
"prompt_number": 8
},
{
"cell_type": "code",
"collapsed": false,
"input": [
"dview.push(dict(a=a))"
],
"language": "python",
"metadata": {},
"outputs": [
{
"output_type": "pyout",
"prompt_number": 9,
"text": [
"<AsyncResult: _push>"
]
}
],
"prompt_number": 9
},
{
"cell_type": "code",
"collapsed": false,
"input": [
"with dview.sync_imports():\n",
" import numpy"
],
"language": "python",
"metadata": {},
"outputs": [
{
"output_type": "stream",
"stream": "stdout",
"text": [
"importing numpy on engine(s)\n"
]
}
],
"prompt_number": 10
},
{
"cell_type": "code",
"collapsed": false,
"input": [
"result = dview.map(parallel_inner, [2, 3], [9, 15])"
],
"language": "python",
"metadata": {},
"outputs": [],
"prompt_number": 11
},
{
"cell_type": "code",
"collapsed": false,
"input": [
"sum(result)"
],
"language": "python",
"metadata": {},
"outputs": [
{
"output_type": "pyout",
"prompt_number": 12,
"text": [
"55.149183935285443"
]
}
],
"prompt_number": 12
},
{
"cell_type": "code",
"collapsed": false,
"input": [
"import ipy_parallel"
],
"language": "python",
"metadata": {},
"outputs": [],
"prompt_number": 13
},
{
"cell_type": "code",
"collapsed": false,
"input": [
"ipy_parallel.some_inner(dview, a)"
],
"language": "python",
"metadata": {},
"outputs": [
{
"ename": "RemoteError",
"evalue": "NameError(global name 'mat' is not defined)",
"output_type": "pyerr",
"traceback": [
"\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
"\u001b[1;31mNameError\u001b[0m Traceback (most recent call last)\u001b[1;32m<string>\u001b[0m in \u001b[0;36m<module>\u001b[1;34m()\u001b[0m",
"\u001b[1;32m/home/moritz/CodeBase/Development/biologis/applications/ipy_parallel.pyc\u001b[0m in \u001b[0;36mparallel_inner\u001b[1;34m(i, j)\u001b[0m",
"\u001b[0;32m 15\u001b[0m \u001b[1;32mdef\u001b[0m \u001b[0mparallel_inner\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mi\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mj\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m",
"\u001b[0;32m 16\u001b[0m \u001b[1;31m# access to global variable `mat` on engine\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m",
"\u001b[1;32m---> 17\u001b[1;33m \u001b[1;32mreturn\u001b[0m \u001b[0mnumpy\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0minner\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mmat\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mi\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mmat\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mj\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m",
"\u001b[0m\u001b[0;32m 18\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m",
"\u001b[0;32m 19\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m",
"\u001b[1;31mNameError\u001b[0m: global name 'mat' is not defined"
]
}
],
"prompt_number": 14
}
],
"metadata": {}
}
]
}
More information about the IPython-User
mailing list