In [30]:
from fireworks import LaunchPad
In [31]:
lp = LaunchPad( host='localhost',
           port=27018,
           name='fireworks-jhoermann',
           username='fireworks',
           password='fireworks' )
In [166]:
import sys
In [167]:
sys.version
Out[167]:
'3.6.8 (default, Jan 14 2019, 11:02:34) \n[GCC 8.0.1 20180414 (experimental) [trunk revision 259383]]'
In [168]:
import os
In [169]:
from monty.serialization import loadfn
In [190]:
from monty.os.path import zpath
In [196]:
from fireworks.utilities.fw_serializers import reconstitute_dates
In [170]:
from datetime import datetime
In [171]:
import tempfile
In [172]:
launch_id = 12392
In [173]:
m_launch = lp.get_launch_by_id(launch_id)
In [174]:
ping_loc = os.path.join(m_launch.launch_dir, "FW_ping.json")
In [175]:
ping_loc
Out[175]:
'/work/ws/nemo/fr_jh1130-fw_ws_20190311-0/launchpad/block_2019-06-30-13-07-21-802466/launcher_2019-07-24-12-30-29-794594/FW_ping.json'
In [176]:
with tempfile.NamedTemporaryFile(mode='w+',suffix='.json') as f:
    f.write('{"ping_time": "2019-07-28T12:54:43.213215"}')
    f.seek(0)
    ping_dict = loadfn(f.name)
In [177]:
ping_dict
Out[177]:
{'ping_time': '2019-07-28T12:54:43.213215'}
In [178]:
type(ping_dict['ping_time'])
Out[178]:
str
In [179]:
ptime = ping_dict['ping_time']
In [180]:
update_time = ptime
In [181]:
update_time = update_time or datetime.utcnow()
In [182]:
update_time
Out[182]:
'2019-07-28T12:54:43.213215'
In [183]:
type(update_time)
Out[183]:
str
In [184]:
m_launch.state_history
Out[184]:
[{'state': 'RESERVED',
  'created_on': datetime.datetime(2019, 7, 24, 12, 30, 29, 747205),
  'updated_on': datetime.datetime(2019, 7, 24, 12, 30, 29, 747211),
  'reservation_id': '6061670'},
 {'state': 'RUNNING',
  'created_on': datetime.datetime(2019, 7, 24, 12, 54, 41, 31150),
  'updated_on': datetime.datetime(2019, 7, 31, 22, 25, 31, 574087),
  'checkpoint': {'_task_n': 0,
   '_all_stored_data': {},
   '_all_update_spec': {},
   '_all_mod_spec': []}},
 {'state': 'FIZZLED',
  'created_on': datetime.datetime(2019, 7, 31, 22, 29, 38, 640869),
  'checkpoint': {'_task_n': 0,
   '_all_stored_data': {},
   '_all_update_spec': {},
   '_all_mod_spec': []}},
 {'state': 'RUNNING',
  'created_on': datetime.datetime(2019, 7, 24, 12, 54, 41, 31150),
  'checkpoint': {'_task_n': 0,
   '_all_stored_data': {},
   '_all_update_spec': {},
   '_all_mod_spec': []},
  'updated_on': datetime.datetime(2019, 8, 6, 19, 5, 59, 924431)}]
In [185]:
m_launch.touch_history(ptime,checkpoint=None)
In [186]:
m_launch.state_history
Out[186]:
[{'state': 'RESERVED',
  'created_on': datetime.datetime(2019, 7, 24, 12, 30, 29, 747205),
  'updated_on': datetime.datetime(2019, 7, 24, 12, 30, 29, 747211),
  'reservation_id': '6061670'},
 {'state': 'RUNNING',
  'created_on': datetime.datetime(2019, 7, 24, 12, 54, 41, 31150),
  'updated_on': datetime.datetime(2019, 7, 31, 22, 25, 31, 574087),
  'checkpoint': {'_task_n': 0,
   '_all_stored_data': {},
   '_all_update_spec': {},
   '_all_mod_spec': []}},
 {'state': 'FIZZLED',
  'created_on': datetime.datetime(2019, 7, 31, 22, 29, 38, 640869),
  'checkpoint': {'_task_n': 0,
   '_all_stored_data': {},
   '_all_update_spec': {},
   '_all_mod_spec': []}},
 {'state': 'RUNNING',
  'created_on': datetime.datetime(2019, 7, 24, 12, 54, 41, 31150),
  'checkpoint': {'_task_n': 0,
   '_all_stored_data': {},
   '_all_update_spec': {},
   '_all_mod_spec': []},
  'updated_on': '2019-07-28T12:54:43.213215'}]
In [187]:
m_launch.to_db_dict()['state_history']
Out[187]:
[{'state': 'RESERVED',
  'created_on': '2019-07-24T12:30:29.747205',
  'updated_on': '2019-07-24T12:30:29.747211',
  'reservation_id': '6061670'},
 {'state': 'RUNNING',
  'created_on': '2019-07-24T12:54:41.031150',
  'updated_on': '2019-07-31T22:25:31.574087',
  'checkpoint': {'_task_n': 0,
   '_all_stored_data': {},
   '_all_update_spec': {},
   '_all_mod_spec': []}},
 {'state': 'FIZZLED',
  'created_on': '2019-07-31T22:29:38.640869',
  'checkpoint': {'_task_n': 0,
   '_all_stored_data': {},
   '_all_update_spec': {},
   '_all_mod_spec': []}},
 {'state': 'RUNNING',
  'created_on': '2019-07-24T12:54:41.031150',
  'checkpoint': {'_task_n': 0,
   '_all_stored_data': {},
   '_all_update_spec': {},
   '_all_mod_spec': []},
  'updated_on': '2019-07-28T12:54:43.213215'}]
In [188]:
lp.launches.update_one(
    {'launch_id': launch_id, 'state': 'RUNNING'},
    {'$set': {
        'state_history': m_launch.to_db_dict()['state_history'],
        'trackers': [t.to_dict() for t in m_launch.trackers]}
    }
)
Out[188]:
<pymongo.results.UpdateResult at 0x7f420900a908>
In [191]:
offline_loc = zpath(os.path.join(m_launch.launch_dir,
                                             "FW_offline.json"))
In [192]:
offline_loc
Out[192]:
'/work/ws/nemo/fr_jh1130-fw_ws_20190311-0/launchpad/block_2019-06-30-13-07-21-802466/launcher_2019-07-24-12-30-29-794594/FW_offline.json'
In [193]:
with tempfile.NamedTemporaryFile(mode='w+',suffix='.json') as f:
    f.write('{"launch_id": 12392, "started_on": "2019-07-24T12:54:41.031150", "checkpoint": {"_task_n": 0, "_all_stored_data": {}, "_all_update_spec": {}, "_all_mod_spec": []}}')
    f.seek(0)
    offline_data = loadfn(f.name)
In [194]:
offline_data
Out[194]:
{'launch_id': 12392,
 'started_on': '2019-07-24T12:54:41.031150',
 'checkpoint': {'_task_n': 0,
  '_all_stored_data': {},
  '_all_update_spec': {},
  '_all_mod_spec': []}}
In [197]:
m_launch.state = 'RUNNING'
for s in m_launch.state_history:
    if s['state'] == 'RUNNING':
        s['created_on'] = reconstitute_dates(offline_data['started_on'])
In [198]:
m_launch.state_history
Out[198]:
[{'state': 'RESERVED',
  'created_on': datetime.datetime(2019, 7, 24, 12, 30, 29, 747205),
  'updated_on': datetime.datetime(2019, 7, 24, 12, 30, 29, 747211),
  'reservation_id': '6061670'},
 {'state': 'RUNNING',
  'created_on': datetime.datetime(2019, 7, 24, 12, 54, 41, 31150),
  'updated_on': datetime.datetime(2019, 7, 31, 22, 25, 31, 574087),
  'checkpoint': {'_task_n': 0,
   '_all_stored_data': {},
   '_all_update_spec': {},
   '_all_mod_spec': []}},
 {'state': 'FIZZLED',
  'created_on': datetime.datetime(2019, 7, 31, 22, 29, 38, 640869),
  'checkpoint': {'_task_n': 0,
   '_all_stored_data': {},
   '_all_update_spec': {},
   '_all_mod_spec': []}},
 {'state': 'RUNNING',
  'created_on': datetime.datetime(2019, 7, 24, 12, 54, 41, 31150),
  'checkpoint': {'_task_n': 0,
   '_all_stored_data': {},
   '_all_update_spec': {},
   '_all_mod_spec': []},
  'updated_on': '2019-07-28T12:54:43.213215'}]
In [199]:
l = lp.launches.find_one_and_replace({'launch_id': m_launch.launch_id},
                                                       m_launch.to_db_dict(), upsert=True)
In [200]:
fw_id = l['fw_id']
In [201]:
fw_id
Out[201]:
16292
In [202]:
f = lp.fireworks.find_one_and_update(
    {'fw_id': fw_id},
       {'$set':
            {'state': 'RUNNING',
             'updated_on': datetime.utcnow()
             }
        }
)
In [204]:
lp._refresh_wf(fw_id)
In [205]:
m_launch.touch_history(checkpoint=offline_data['checkpoint'])
In [206]:
m_launch.state_history
Out[206]:
[{'state': 'RESERVED',
  'created_on': datetime.datetime(2019, 7, 24, 12, 30, 29, 747205),
  'updated_on': datetime.datetime(2019, 7, 24, 12, 30, 29, 747211),
  'reservation_id': '6061670'},
 {'state': 'RUNNING',
  'created_on': datetime.datetime(2019, 7, 24, 12, 54, 41, 31150),
  'updated_on': datetime.datetime(2019, 7, 31, 22, 25, 31, 574087),
  'checkpoint': {'_task_n': 0,
   '_all_stored_data': {},
   '_all_update_spec': {},
   '_all_mod_spec': []}},
 {'state': 'FIZZLED',
  'created_on': datetime.datetime(2019, 7, 31, 22, 29, 38, 640869),
  'checkpoint': {'_task_n': 0,
   '_all_stored_data': {},
   '_all_update_spec': {},
   '_all_mod_spec': []}},
 {'state': 'RUNNING',
  'created_on': datetime.datetime(2019, 7, 24, 12, 54, 41, 31150),
  'checkpoint': {'_task_n': 0,
   '_all_stored_data': {},
   '_all_update_spec': {},
   '_all_mod_spec': []},
  'updated_on': datetime.datetime(2019, 8, 6, 20, 10, 25, 918353)}]
In [207]:
lp.launches.find_one_and_replace(
    {'launch_id': m_launch.launch_id},
    m_launch.to_db_dict(), upsert=True)
Out[207]:
{'_id': ObjectId('5d384f659d3a40f69aa7f59a'),
 'fworker': {'name': 'nemo_queue_worker',
  'category': ['nemo_queue_offline'],
  'query': '{}',
  'env': {'lmp': 'module purge; module use /work/ws/nemo/fr_lp1029-IMTEK_SIMULATION-0/modulefiles; module load lammps/16Mar18-gnu-7.3-openmpi-3.1-colvars-09Feb19; mpirun ${MPIRUN_OPTIONS} lmp',
   'exchange_substrate.py': 'module purge; module use /work/ws/nemo/fr_lp1029-IMTEK_SIMULATION-0/modulefiles; module load mdtools ovitos; exchange_substrate.py',
   'extract_bb.py': 'module purge; module use /work/ws/nemo/fr_lp1029-IMTEK_SIMULATION-0/modulefiles; module load mdtools/12Mar19-python-2.7; extract_bb.py',
   'extract_indenter_nonindenter_forces_from_netcdf.py': 'module purge; module use /work/ws/nemo/fr_lp1029-IMTEK_SIMULATION-0/modulefiles; module load mdtools/11Jul19; extract_indenter_nonindenter_forces_from_netcdf.py',
   'extract_property.py': 'module purge; module use /work/ws/nemo/fr_lp1029-IMTEK_SIMULATION-0/modulefiles; module load mdtools ovitos; extract_property.py',
   'extract_thermo.sh': 'module purge; module use /work/ws/nemo/fr_lp1029-IMTEK_SIMULATION-0/modulefiles; module load mdtools; extract_thermo.sh',
   'join_thermo.py': 'module purge; module use /work/ws/nemo/fr_lp1029-IMTEK_SIMULATION-0/modulefiles; module load mdtools; join_thermo.py',
   'merge.py': 'module purge; module use /work/ws/nemo/fr_lp1029-IMTEK_SIMULATION-0/modulefiles; module load mdtools/12Mar19-python-2.7; merge.py',
   'ncfilter.py': 'module purge; module use /work/ws/nemo/fr_lp1029-IMTEK_SIMULATION-0/modulefiles; module load mdtools/11Jul19; mpirun ${MPIRUN_OPTIONS} ncfilter.py',
   'ncjoin.py': 'module purge; module use /work/ws/nemo/fr_lp1029-IMTEK_SIMULATION-0/modulefiles; module load mdtools; ncjoin.py',
   'pizza.py': 'module purge; module use /work/ws/nemo/fr_lp1029-IMTEK_SIMULATION-0/modulefiles; module load mdtools/12Mar19-python-2.7; pizza.py',
   'strip_comments.py': 'module purge; module use /work/ws/nemo/fr_lp1029-IMTEK_SIMULATION-0/modulefiles; module load mdtools/12Mar19-python-2.7; strip_comments.py',
   'to_hybrid.py': 'module purge; module use /work/ws/nemo/fr_lp1029-IMTEK_SIMULATION-0/modulefiles; module load mdtools; to_hybrid.py',
   'vmd': 'module purge; module use /work/ws/nemo/fr_lp1029-IMTEK_SIMULATION-0/modulefiles; module load vmd/1.9.3-text; vmd',
   'smbsync.py': 'module purge; module use /work/ws/nemo/fr_lp1029-IMTEK_SIMULATION-0/modulefiles; module load mdtools; smbsync.py'}},
 'fw_id': 16292,
 'launch_dir': '/work/ws/nemo/fr_jh1130-fw_ws_20190311-0/launchpad/block_2019-06-30-13-07-21-802466/launcher_2019-07-24-12-30-29-794594',
 'host': 'login2.nemo.privat',
 'ip': '10.16.44.2',
 'trackers': [{'filename': 'log.lammps', 'nlines': 25, 'allow_zipped': False}],
 'action': None,
 'state': 'RUNNING',
 'state_history': [{'state': 'RESERVED',
   'created_on': '2019-07-24T12:30:29.747205',
   'updated_on': '2019-07-24T12:30:29.747211',
   'reservation_id': '6061670'},
  {'state': 'RUNNING',
   'created_on': '2019-07-24T12:54:41.031150',
   'updated_on': '2019-07-31T22:25:31.574087',
   'checkpoint': {'_task_n': 0,
    '_all_stored_data': {},
    '_all_update_spec': {},
    '_all_mod_spec': []}},
  {'state': 'FIZZLED',
   'created_on': '2019-07-31T22:29:38.640869',
   'checkpoint': {'_task_n': 0,
    '_all_stored_data': {},
    '_all_update_spec': {},
    '_all_mod_spec': []}},
  {'state': 'RUNNING',
   'created_on': '2019-07-24T12:54:41.031150',
   'checkpoint': {'_task_n': 0,
    '_all_stored_data': {},
    '_all_update_spec': {},
    '_all_mod_spec': []},
   'updated_on': '2019-07-28T12:54:43.213215'}],
 'launch_id': 12392,
 'time_start': '2019-07-24T12:54:41.031150',
 'time_end': '2019-07-31T22:29:38.640869',
 'runtime_secs': 639297.609719,
 'reservedtime_secs': 1451.283945}