Adjusted sql aggregation to add values in chunks of 500 columns.

ih/statistics.py

@@ -27,6 +27,7 @@ import json
 import traceback
 import shutil
 import subprocess
+import itertools
 from scipy import stats
 
 class Stats:
@@ -217,14 +218,18 @@ class Stats:
         for i,f in enumerate(dblist):
             if os.path.isfile(f):
                 conn = self._openConnection(f)
-                cols = self._getHeaders("images", conn)
-                for col in cols:
-                    self._addColumn(col, "images")
-                result = conn.execute("select pegasusid," + ",".join(cols) + " from images")
-                query = "update images set " + ",".join([col + "=?" for col in cols]) + " where pegasusid=?"
-                values = [tuple([row[col] for col in cols] + [row["pegasusid"]]) for row in result]
-                self.conn.executemany(query, values)
-                self.conn.commit()
+                headers = self._getHeaders("images", conn)
+                col_set = itertools.islice(headers, 500)
+                while col_set:
+                    cols = list(col_set)
+                    for col in cols:
+                        self._addColumn(col, "images")
+                    result = conn.execute("select pegasusid," + ",".join(cols) + " from images")
+                    query = "update images set " + ",".join([col + "=?" for col in cols]) + " where pegasusid=?"
+                    values = [tuple([row[col] for col in cols] + [row["pegasusid"]]) for row in result]
+                    self.conn.executemany(query, values)
+                    self.conn.commit()
+                    col_set = itertools.islice(headers, 500)
                 self._closeConnection(conn)
             else:
                 print "DB File: '%s' does not exist." % (f,)