Details
-
Bug
-
Status: Resolved
-
Major
-
Resolution: Fixed
-
master, 5.0.0
-
None
Description
A user reported an issue where pegasus-analyzer was not showing the right results for a failed workflow
On investigating it , it seems pegasus-db-admin failed to upgrade the database as monitord.log has these errors
2021-04-29 12:55:11,134:WARNING:Pegasus.monitoring.workflow(169): DASHBOARD DB NL-LOAD-ERROR --> e9191f06-56d5-498a-95da-afeb9cbef13c - pipeline-0
2021-04-29 12:55:11,134:WARNING:Pegasus.monitoring.workflow(180): error sending event to dashboard db: xwf.start --> {'xwf__id': 'e9191f06-56d5-498a-95da-afeb9cbef13c', 'ts': 1619716371.0, 'reason': None, 'restart_count': 0, 'root__xwf__id': 'e9191f06-56d5-498a-95da-afeb9cbef13c', 'dax': '/lustre/work/deogun/npavlovikj/FFH/ProkEvo/root-pipeline.dax', 'dag': 'pipeline-0.dag', 'wf__ts': 1619716365, 'submit__dir': '/work/deogun/npavlovikj/FFH/ProkEvo/npavlovikj/pegasus/pipeline/20210429T121245-0500', 'pegasus__version': '5.0.0', 'submit__hostname': 'login.crane.hcc.unl.edu', 'wf__user': 'npavlovikj'}
2021-04-29 12:55:11,139:WARNING:Pegasus.monitoring.workflow(183): Traceback (most recent call last):
File "/util/opt/pegasus-wms/5.0/lib64/pegasus/externals/python/sqlalchemy/engine/base.py", line 1247, in _execute_context
self.dialect.do_execute(
File "/util/opt/pegasus-wms/5.0/lib64/pegasus/externals/python/sqlalchemy/engine/default.py", line 588, in do_execute
cursor.execute(statement, parameters)
sqlite3.OperationalError: no such table: main._master_workflow_old
The above exception was the direct cause of the following exception:
Traceback (most recent call last):
File "/util/opt/pegasus-wms/5.0/lib64/python3.6/site-packages/Pegasus/db/dashboard_loader.py", line 215, in hard_flush
self.session.commit()
File "/util/opt/pegasus-wms/5.0/lib64/pegasus/externals/python/sqlalchemy/orm/scoping.py", line 162, in do
return getattr(self.registry(), name)(*args, **kwargs)
File "/util/opt/pegasus-wms/5.0/lib64/pegasus/externals/python/sqlalchemy/orm/session.py", line 1036, in commit
self.transaction.commit()
File "/util/opt/pegasus-wms/5.0/lib64/pegasus/externals/python/sqlalchemy/orm/session.py", line 503, in commit
self._prepare_impl()
File "/util/opt/pegasus-wms/5.0/lib64/pegasus/externals/python/sqlalchemy/orm/session.py", line 482, in _prepare_impl
self.session.flush()
File "/util/opt/pegasus-wms/5.0/lib64/pegasus/externals/python/sqlalchemy/orm/session.py", line 2496, in flush
self._flush(objects)
File "/util/opt/pegasus-wms/5.0/lib64/pegasus/externals/python/sqlalchemy/orm/session.py", line 2637, in _flush
transaction.rollback(_capture_exception=True)
File "/util/opt/pegasus-wms/5.0/lib64/pegasus/externals/python/sqlalchemy/util/langhelpers.py", line 68, in __exit__
compat.raise_(
File "/util/opt/pegasus-wms/5.0/lib64/pegasus/externals/python/sqlalchemy/util/compat.py", line 178, in raise_
raise exception
File "/util/opt/pegasus-wms/5.0/lib64/pegasus/externals/python/sqlalchemy/orm/session.py", line 2597, in _flush
flush_context.execute()
File "/util/opt/pegasus-wms/5.0/lib64/pegasus/externals/python/sqlalchemy/orm/unitofwork.py", line 422, in execute
rec.execute(self)
File "/util/opt/pegasus-wms/5.0/lib64/pegasus/externals/python/sqlalchemy/orm/unitofwork.py", line 586, in execute
persistence.save_obj(
File "/util/opt/pegasus-wms/5.0/lib64/pegasus/externals/python/sqlalchemy/orm/persistence.py", line 239, in save_obj
_emit_insert_statements(
File "/util/opt/pegasus-wms/5.0/lib64/pegasus/externals/python/sqlalchemy/orm/persistence.py", line 1083, in _emit_insert_statements
c = cached_connections[connection].execute(statement, multiparams)
File "/util/opt/pegasus-wms/5.0/lib64/pegasus/externals/python/sqlalchemy/engine/base.py", line 984, in execute
return meth(self, multiparams, params)
File "/util/opt/pegasus-wms/5.0/lib64/pegasus/externals/python/sqlalchemy/sql/elements.py", line 293, in _execute_on_connection
return connection._execute_clauseelement(self, multiparams, params)
File "/util/opt/pegasus-wms/5.0/lib64/pegasus/externals/python/sqlalchemy/engine/base.py", line 1097, in _execute_clauseelement
ret = self._execute_context(
File "/util/opt/pegasus-wms/5.0/lib64/pegasus/externals/python/sqlalchemy/engine/base.py", line 1287, in _execute_context
self._handle_dbapi_exception(
File "/util/opt/pegasus-wms/5.0/lib64/pegasus/externals/python/sqlalchemy/engine/base.py", line 1481, in _handle_dbapi_exception
util.raise_(
File "/util/opt/pegasus-wms/5.0/lib64/pegasus/externals/python/sqlalchemy/util/compat.py", line 178, in raise_
raise exception
File "/util/opt/pegasus-wms/5.0/lib64/pegasus/externals/python/sqlalchemy/engine/base.py", line 1247, in _execute_context
self.dialect.do_execute(
File "/util/opt/pegasus-wms/5.0/lib64/pegasus/externals/python/sqlalchemy/engine/default.py", line 588, in do_execute
cursor.execute(statement, parameters)
sqlalchemy.exc.OperationalError: (sqlite3.OperationalError) no such table: main._master_workflow_old
On investigating it , it seems pegasus-db-admin failed to upgrade the database as monitord.log has these errors
2021-04-29 12:55:11,134:WARNING:Pegasus.monitoring.workflow(169): DASHBOARD DB NL-LOAD-ERROR --> e9191f06-56d5-498a-95da-afeb9cbef13c - pipeline-0
2021-04-29 12:55:11,134:WARNING:Pegasus.monitoring.workflow(180): error sending event to dashboard db: xwf.start --> {'xwf__id': 'e9191f06-56d5-498a-95da-afeb9cbef13c', 'ts': 1619716371.0, 'reason': None, 'restart_count': 0, 'root__xwf__id': 'e9191f06-56d5-498a-95da-afeb9cbef13c', 'dax': '/lustre/work/deogun/npavlovikj/FFH/ProkEvo/root-pipeline.dax', 'dag': 'pipeline-0.dag', 'wf__ts': 1619716365, 'submit__dir': '/work/deogun/npavlovikj/FFH/ProkEvo/npavlovikj/pegasus/pipeline/20210429T121245-0500', 'pegasus__version': '5.0.0', 'submit__hostname': 'login.crane.hcc.unl.edu', 'wf__user': 'npavlovikj'}
2021-04-29 12:55:11,139:WARNING:Pegasus.monitoring.workflow(183): Traceback (most recent call last):
File "/util/opt/pegasus-wms/5.0/lib64/pegasus/externals/python/sqlalchemy/engine/base.py", line 1247, in _execute_context
self.dialect.do_execute(
File "/util/opt/pegasus-wms/5.0/lib64/pegasus/externals/python/sqlalchemy/engine/default.py", line 588, in do_execute
cursor.execute(statement, parameters)
sqlite3.OperationalError: no such table: main._master_workflow_old
The above exception was the direct cause of the following exception:
Traceback (most recent call last):
File "/util/opt/pegasus-wms/5.0/lib64/python3.6/site-packages/Pegasus/db/dashboard_loader.py", line 215, in hard_flush
self.session.commit()
File "/util/opt/pegasus-wms/5.0/lib64/pegasus/externals/python/sqlalchemy/orm/scoping.py", line 162, in do
return getattr(self.registry(), name)(*args, **kwargs)
File "/util/opt/pegasus-wms/5.0/lib64/pegasus/externals/python/sqlalchemy/orm/session.py", line 1036, in commit
self.transaction.commit()
File "/util/opt/pegasus-wms/5.0/lib64/pegasus/externals/python/sqlalchemy/orm/session.py", line 503, in commit
self._prepare_impl()
File "/util/opt/pegasus-wms/5.0/lib64/pegasus/externals/python/sqlalchemy/orm/session.py", line 482, in _prepare_impl
self.session.flush()
File "/util/opt/pegasus-wms/5.0/lib64/pegasus/externals/python/sqlalchemy/orm/session.py", line 2496, in flush
self._flush(objects)
File "/util/opt/pegasus-wms/5.0/lib64/pegasus/externals/python/sqlalchemy/orm/session.py", line 2637, in _flush
transaction.rollback(_capture_exception=True)
File "/util/opt/pegasus-wms/5.0/lib64/pegasus/externals/python/sqlalchemy/util/langhelpers.py", line 68, in __exit__
compat.raise_(
File "/util/opt/pegasus-wms/5.0/lib64/pegasus/externals/python/sqlalchemy/util/compat.py", line 178, in raise_
raise exception
File "/util/opt/pegasus-wms/5.0/lib64/pegasus/externals/python/sqlalchemy/orm/session.py", line 2597, in _flush
flush_context.execute()
File "/util/opt/pegasus-wms/5.0/lib64/pegasus/externals/python/sqlalchemy/orm/unitofwork.py", line 422, in execute
rec.execute(self)
File "/util/opt/pegasus-wms/5.0/lib64/pegasus/externals/python/sqlalchemy/orm/unitofwork.py", line 586, in execute
persistence.save_obj(
File "/util/opt/pegasus-wms/5.0/lib64/pegasus/externals/python/sqlalchemy/orm/persistence.py", line 239, in save_obj
_emit_insert_statements(
File "/util/opt/pegasus-wms/5.0/lib64/pegasus/externals/python/sqlalchemy/orm/persistence.py", line 1083, in _emit_insert_statements
c = cached_connections[connection].execute(statement, multiparams)
File "/util/opt/pegasus-wms/5.0/lib64/pegasus/externals/python/sqlalchemy/engine/base.py", line 984, in execute
return meth(self, multiparams, params)
File "/util/opt/pegasus-wms/5.0/lib64/pegasus/externals/python/sqlalchemy/sql/elements.py", line 293, in _execute_on_connection
return connection._execute_clauseelement(self, multiparams, params)
File "/util/opt/pegasus-wms/5.0/lib64/pegasus/externals/python/sqlalchemy/engine/base.py", line 1097, in _execute_clauseelement
ret = self._execute_context(
File "/util/opt/pegasus-wms/5.0/lib64/pegasus/externals/python/sqlalchemy/engine/base.py", line 1287, in _execute_context
self._handle_dbapi_exception(
File "/util/opt/pegasus-wms/5.0/lib64/pegasus/externals/python/sqlalchemy/engine/base.py", line 1481, in _handle_dbapi_exception
util.raise_(
File "/util/opt/pegasus-wms/5.0/lib64/pegasus/externals/python/sqlalchemy/util/compat.py", line 178, in raise_
raise exception
File "/util/opt/pegasus-wms/5.0/lib64/pegasus/externals/python/sqlalchemy/engine/base.py", line 1247, in _execute_context
self.dialect.do_execute(
File "/util/opt/pegasus-wms/5.0/lib64/pegasus/externals/python/sqlalchemy/engine/default.py", line 588, in do_execute
cursor.execute(statement, parameters)
sqlalchemy.exc.OperationalError: (sqlite3.OperationalError) no such table: main._master_workflow_old