[Numpy-svn] r8715 - in trunk/numpy/lib: . tests

numpy-svn@scip... numpy-svn@scip...
Mon Sep 13 18:54:54 CDT 2010


Author: pierregm
Date: 2010-09-13 18:54:54 -0500 (Mon, 13 Sep 2010)
New Revision: 8715

Modified:
   trunk/numpy/lib/npyio.py
   trunk/numpy/lib/tests/test_io.py
Log:
* fixed the behavior of {{{skip_footer}}} in {{{genfromtxt}}} when some invalid lines are present (bug #1593)

Modified: trunk/numpy/lib/npyio.py
===================================================================
--- trunk/numpy/lib/npyio.py	2010-09-13 15:43:27 UTC (rev 8714)
+++ trunk/numpy/lib/npyio.py	2010-09-13 23:54:54 UTC (rev 8715)
@@ -1343,10 +1343,10 @@
             try:
                 values = [values[_] for _ in usecols]
             except IndexError:
-                append_to_invalid((i, nbvalues))
+                append_to_invalid((i + skip_header + 1, nbvalues))
                 continue
         elif nbvalues != nbcols:
-            append_to_invalid((i, nbvalues))
+            append_to_invalid((i + skip_header + 1, nbvalues))
             continue
         # Store the values
         append_to_rows(tuple(values))
@@ -1354,11 +1354,6 @@
             append_to_masks(tuple([v.strip() in m
                                    for (v, m) in zip(values, missing_values)]))
 
-    # Strip the last skip_footer data
-    if skip_footer > 0:
-        rows = rows[:-skip_footer]
-        if usemask:
-            masks = masks[:-skip_footer]
 
     # Upgrade the converters (if needed)
     if dtype is None:
@@ -1378,17 +1373,23 @@
                         raise ConverterError(errmsg)
 
     # Check that we don't have invalid values
-    if len(invalid) > 0:
-        nbrows = len(rows)
+    nbinvalid = len(invalid)
+    if nbinvalid > 0:
+        nbrows = len(rows) + nbinvalid - skip_footer
         # Construct the error message
         template = "    Line #%%i (got %%i columns instead of %i)" % nbcols
         if skip_footer > 0:
-            nbrows -= skip_footer
-            errmsg = [template % (i + skip_header + 1, nb)
-                      for (i, nb) in invalid if i < nbrows]
-        else:
-            errmsg = [template % (i + skip_header + 1, nb)
-                      for (i, nb) in invalid]
+            nbinvalid_skipped = len([_ for _ in invalid
+                                     if _[0] > nbrows + skip_header])
+            invalid = invalid[:nbinvalid - nbinvalid_skipped]
+            skip_footer -= nbinvalid_skipped
+#            
+#            nbrows -= skip_footer
+#            errmsg = [template % (i, nb)
+#                      for (i, nb) in invalid if i < nbrows]
+#        else:
+        errmsg = [template % (i, nb)
+                  for (i, nb) in invalid]
         if len(errmsg):
             errmsg.insert(0, "Some errors were detected !")
             errmsg = "\n".join(errmsg)
@@ -1399,6 +1400,13 @@
             else:
                 warnings.warn(errmsg, ConversionWarning)
 
+    # Strip the last skip_footer data
+    if skip_footer > 0:
+        rows = rows[:-skip_footer]
+        if usemask:
+            masks = masks[:-skip_footer]
+
+
     # Convert each value according to the converter:
     # We want to modify the list in place to avoid creating a new one...
 #    if loose:

Modified: trunk/numpy/lib/tests/test_io.py
===================================================================
--- trunk/numpy/lib/tests/test_io.py	2010-09-13 15:43:27 UTC (rev 8714)
+++ trunk/numpy/lib/tests/test_io.py	2010-09-13 23:54:54 UTC (rev 8715)
@@ -539,10 +539,33 @@
         data[-1] = "99,99"
         kwargs = dict(delimiter=",", names=True, skip_header=5, skip_footer=10)
         test = np.genfromtxt(StringIO(asbytes("\n".join(data))), **kwargs)
-        ctrl = np.array([("%f" % i, "%f" % i, "%f" % i) for i in range(40)],
+        ctrl = np.array([("%f" % i, "%f" % i, "%f" % i) for i in range(41)],
                         dtype=[(_, float) for _ in "ABC"])
         assert_equal(test, ctrl)
 
+    def test_skip_footer_with_invalid(self):
+        import warnings
+        basestr = '1 1\n2 2\n3 3\n4 4\n5  \n6  \n7  \n'
+        warnings.filterwarnings("ignore")
+        # Footer too small to get rid of all invalid values 
+        assert_raises(ValueError, np.genfromtxt,
+                      StringIO(basestr), skip_footer=1)
+#        except ValueError:
+#            pass
+        a = np.genfromtxt(StringIO(basestr), skip_footer=1, invalid_raise=False)
+        assert_equal(a, np.array([[1., 1.], [2., 2.], [3., 3.], [4., 4.]]))
+        #
+        a = np.genfromtxt(StringIO(basestr), skip_footer=3)
+        assert_equal(a, np.array([[1., 1.], [2., 2.], [3., 3.], [4., 4.]]))
+        #
+        basestr = '1 1\n2  \n3 3\n4 4\n5  \n6 6\n7 7\n'
+        a = np.genfromtxt(StringIO(basestr), skip_footer=1, invalid_raise=False)
+        assert_equal(a, np.array([[1., 1.], [3., 3.], [4., 4.], [6., 6.]]))
+        a = np.genfromtxt(StringIO(basestr), skip_footer=3, invalid_raise=False)
+        assert_equal(a, np.array([[1., 1.], [3., 3.], [4., 4.]]))
+        warnings.resetwarnings()
+
+
     def test_header(self):
         "Test retrieving a header"
         data = StringIO('gender age weight\nM 64.0 75.0\nF 25.0 60.0')



More information about the Numpy-svn mailing list