Commit 83c257b1 authored by aknecht2's avatar aknecht2

Adjusted sql aggregation to add values in chunks of 500 columns.

parent dcbd56c8
......@@ -27,6 +27,7 @@ import json
import traceback
import shutil
import subprocess
import itertools
from scipy import stats
class Stats:
......@@ -217,14 +218,18 @@ class Stats:
for i,f in enumerate(dblist):
if os.path.isfile(f):
conn = self._openConnection(f)
cols = self._getHeaders("images", conn)
for col in cols:
self._addColumn(col, "images")
result = conn.execute("select pegasusid," + ",".join(cols) + " from images")
query = "update images set " + ",".join([col + "=?" for col in cols]) + " where pegasusid=?"
values = [tuple([row[col] for col in cols] + [row["pegasusid"]]) for row in result]
self.conn.executemany(query, values)
self.conn.commit()
headers = self._getHeaders("images", conn)
col_set = itertools.islice(headers, 500)
while col_set:
cols = list(col_set)
for col in cols:
self._addColumn(col, "images")
result = conn.execute("select pegasusid," + ",".join(cols) + " from images")
query = "update images set " + ",".join([col + "=?" for col in cols]) + " where pegasusid=?"
values = [tuple([row[col] for col in cols] + [row["pegasusid"]]) for row in result]
self.conn.executemany(query, values)
self.conn.commit()
col_set = itertools.islice(headers, 500)
self._closeConnection(conn)
else:
print "DB File: '%s' does not exist." % (f,)
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment