Commit 83c257b1 authored by aknecht2's avatar aknecht2
Browse files

Adjusted sql aggregation to add values in chunks of 500 columns.

parent dcbd56c8
...@@ -27,6 +27,7 @@ import json ...@@ -27,6 +27,7 @@ import json
import traceback import traceback
import shutil import shutil
import subprocess import subprocess
import itertools
from scipy import stats from scipy import stats
class Stats: class Stats:
...@@ -217,14 +218,18 @@ class Stats: ...@@ -217,14 +218,18 @@ class Stats:
for i,f in enumerate(dblist): for i,f in enumerate(dblist):
if os.path.isfile(f): if os.path.isfile(f):
conn = self._openConnection(f) conn = self._openConnection(f)
cols = self._getHeaders("images", conn) headers = self._getHeaders("images", conn)
for col in cols: col_set = itertools.islice(headers, 500)
self._addColumn(col, "images") while col_set:
result = conn.execute("select pegasusid," + ",".join(cols) + " from images") cols = list(col_set)
query = "update images set " + ",".join([col + "=?" for col in cols]) + " where pegasusid=?" for col in cols:
values = [tuple([row[col] for col in cols] + [row["pegasusid"]]) for row in result] self._addColumn(col, "images")
self.conn.executemany(query, values) result = conn.execute("select pegasusid," + ",".join(cols) + " from images")
self.conn.commit() query = "update images set " + ",".join([col + "=?" for col in cols]) + " where pegasusid=?"
values = [tuple([row[col] for col in cols] + [row["pegasusid"]]) for row in result]
self.conn.executemany(query, values)
self.conn.commit()
col_set = itertools.islice(headers, 500)
self._closeConnection(conn) self._closeConnection(conn)
else: else:
print "DB File: '%s' does not exist." % (f,) print "DB File: '%s' does not exist." % (f,)
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment