RecursionError: maximum recursion depth exceeded

Hi,

I am currently debugging a workflow, but when I try to rerun FIZZLED FWs, it raises errors : “RecursionError: maximum recursion depth exceeded”

Note that these FWs are in a workflow dynamically generated and append by a custom firetask.

Is there a way to rerun these fireworks without deleting and rebuilding the entire workflow ?

How can I avoid such problems ?

Best regards,
David

Here are some tracebacks :

$ lpad rerun_fws -s FIZZLED

Traceback (most recent call last):

File “/b/home/ipgs/dmichea/venv_a2s/bin/lpad”, line 11, in

load_entry_point(‘FireWorks==1.4.1’, ‘console_scripts’, ‘lpad’)()

File “/b/home/ipgs/dmichea/venv_a2s/lib/python3.6/site-packages/fireworks/scripts/lpad_run.py”, line 1053, in lpad

args.func(args)

File “/b/home/ipgs/dmichea/venv_a2s/lib/python3.6/site-packages/fireworks/scripts/lpad_run.py”, line 427, in rerun_fws

lp.rerun_fw(int(f), clear_recovery=args.clear_recovery)

File “/b/home/ipgs/dmichea/venv_a2s/lib/python3.6/site-packages/fireworks/core/launchpad.py”, line 1260, in rerun_fw

updated_ids = wf.rerun_fw(fw_id)

File “/b/home/ipgs/dmichea/venv_a2s/lib/python3.6/site-packages/fireworks/core/firework.py”, line 863, in rerun_fw

updated_ids = updated_ids.union(self.rerun_fw(child_id, updated_ids))

File “/b/home/ipgs/dmichea/venv_a2s/lib/python3.6/site-packages/fireworks/core/firework.py”, line 863, in rerun_fw

updated_ids = updated_ids.union(self.rerun_fw(child_id, updated_ids))

File “/b/home/ipgs/dmichea/venv_a2s/lib/python3.6/site-packages/fireworks/core/firework.py”, line 863, in rerun_fw

updated_ids = updated_ids.union(self.rerun_fw(child_id, updated_ids))

[Previous line repeated 986 more times]

File “/b/home/ipgs/dmichea/venv_a2s/lib/python3.6/site-packages/fireworks/core/firework.py”, line 858, in rerun_fw

m_fw._rerun()

File “/b/home/ipgs/dmichea/venv_a2s/lib/python3.6/site-packages/fireworks/core/launchpad.py”, line 1592, in _rerun

self.full_fw._rerun()

File “/b/home/ipgs/dmichea/venv_a2s/lib/python3.6/site-packages/fireworks/core/launchpad.py”, line 1692, in full_fw

self._get_launch_data(launch_field)

File “/b/home/ipgs/dmichea/venv_a2s/lib/python3.6/site-packages/fireworks/core/launchpad.py”, line 1707, in _get_launch_data

fw = self.partial_fw # assure stage 1

RecursionError: maximum recursion depth exceeded

$ lpad get_fws -s FIZZLED

[

{

“fw_id”: 165,

“created_on”: “2017-06-30T08:59:50.860965”,

“updated_on”: “2017-06-30T09:00:51.518826”,

“state”: “FIZZLED”,

“name”: “Pre-correlation: 8937 vs 8931”

},

{

“fw_id”: 166,

“created_on”: “2017-06-30T08:59:50.835693”,

“updated_on”: “2017-06-30T09:00:51.520577”,

“state”: “FIZZLED”,

“name”: “Pre-correlation: 8942 vs 8931”

},

{

“fw_id”: 171,

“created_on”: “2017-06-30T08:59:50.834089”,

“updated_on”: “2017-06-30T09:00:51.835825”,

“state”: “FIZZLED”,

“name”: “Pre-correlation: 8964 vs 8931”

},

{

“fw_id”: 177,

“created_on”: “2017-06-30T08:59:50.843329”,

“updated_on”: “2017-06-30T09:00:51.508829”,

“state”: “FIZZLED”,

“name”: “Pre-correlation: 8970 vs 8931”

},

{

“fw_id”: 178,

“created_on”: “2017-06-30T08:59:50.831047”,

“updated_on”: “2017-06-30T09:00:51.522512”,

“state”: “FIZZLED”,

“name”: “Pre-correlation: 8964 vs 8937”

},

{

“fw_id”: 181,

“created_on”: “2017-06-30T08:59:50.841427”,

“updated_on”: “2017-06-30T09:00:51.825738”,

“state”: “FIZZLED”,

“name”: “Pre-correlation: 8970 vs 8937”

}

]

$ lpad rerun_fws -i 181

Traceback (most recent call last):

File “/b/home/ipgs/dmichea/venv_a2s/bin/lpad”, line 11, in

load_entry_point(‘FireWorks==1.4.1’, ‘console_scripts’, ‘lpad’)()

File “/b/home/ipgs/dmichea/venv_a2s/lib/python3.6/site-packages/fireworks/scripts/lpad_run.py”, line 1053, in lpad

args.func(args)

File “/b/home/ipgs/dmichea/venv_a2s/lib/python3.6/site-packages/fireworks/scripts/lpad_run.py”, line 427, in rerun_fws

lp.rerun_fw(int(f), clear_recovery=args.clear_recovery)

File “/b/home/ipgs/dmichea/venv_a2s/lib/python3.6/site-packages/fireworks/core/launchpad.py”, line 1260, in rerun_fw

updated_ids = wf.rerun_fw(fw_id)

File “/b/home/ipgs/dmichea/venv_a2s/lib/python3.6/site-packages/fireworks/core/firework.py”, line 863, in rerun_fw

updated_ids = updated_ids.union(self.rerun_fw(child_id, updated_ids))

File “/b/home/ipgs/dmichea/venv_a2s/lib/python3.6/site-packages/fireworks/core/firework.py”, line 863, in rerun_fw

updated_ids = updated_ids.union(self.rerun_fw(child_id, updated_ids))

File “/b/home/ipgs/dmichea/venv_a2s/lib/python3.6/site-packages/fireworks/core/firework.py”, line 863, in rerun_fw

updated_ids = updated_ids.union(self.rerun_fw(child_id, updated_ids))

[Previous line repeated 986 more times]

File “/b/home/ipgs/dmichea/venv_a2s/lib/python3.6/site-packages/fireworks/core/firework.py”, line 858, in rerun_fw

m_fw._rerun()

File “/b/home/ipgs/dmichea/venv_a2s/lib/python3.6/site-packages/fireworks/core/launchpad.py”, line 1592, in _rerun

self.full_fw._rerun()

File “/b/home/ipgs/dmichea/venv_a2s/lib/python3.6/site-packages/fireworks/core/launchpad.py”, line 1692, in full_fw

self._get_launch_data(launch_field)

File “/b/home/ipgs/dmichea/venv_a2s/lib/python3.6/site-packages/fireworks/core/launchpad.py”, line 1707, in _get_launch_data

fw = self.partial_fw # assure stage 1

RecursionError: maximum recursion depth exceeded

Well, I just figured out som I use custom firetask created to build and append new worflow to the current:

class AppendAttributesTask(FireTaskBase):

_fw_name = 'AppendAttributesTask'

required_params = ["src_id"]

def run_task(self, fw_spec):

	(A2S_logger, __) = A2S_config.init_loggers()

	src_id = self["src_id"]

	attributes_wf = attributes_S2.get_attributes_wf(src_id)

	if attributes_wf:

		A2S_logger.info("create & append attributes computation workflow")

		return FWAction(additions=attributes_wf)

	else:

		A2S_logger.fatal("attributes_S2.get_attributes_wf failed : no workflow created for src id %s" % str(src_id))

		return FWAction()

``

As you can see, I defined a fixed _fw_name for this task. I read in the doc that it have to be unique. I use this task only once in each workflow, but I run several workflows at the same time which use this task.

I suppose that in this case, the name is not unique in the Mongo DB and that the problem from my previous post should come from this fact …

It seems me quite weird that I can not have several instances of a FireTask class in my workflows …

How can I overcome this problem ? Should I define a dynamic name to avoid name collisions (with a randomly generated suffix for example) ?

Best regards,
David

···

Le vendredi 30 juin 2017 11:15:17 UTC+2, [email protected] a écrit :

Hi,

I am currently debugging a workflow, but when I try to rerun FIZZLED FWs, it raises errors : “RecursionError: maximum recursion depth exceeded”

Note that these FWs are in a workflow dynamically generated and append by a custom firetask.

Is there a way to rerun these fireworks without deleting and rebuilding the entire workflow ?

How can I avoid such problems ?

Best regards,
David

Here are some tracebacks :

$ lpad rerun_fws -s FIZZLED

Traceback (most recent call last):

File “/b/home/ipgs/dmichea/venv_a2s/bin/lpad”, line 11, in

load_entry_point(‘FireWorks==1.4.1’, ‘console_scripts’, ‘lpad’)()

File “/b/home/ipgs/dmichea/venv_a2s/lib/python3.6/site-packages/fireworks/scripts/lpad_run.py”, line 1053, in lpad

args.func(args)

File “/b/home/ipgs/dmichea/venv_a2s/lib/python3.6/site-packages/fireworks/scripts/lpad_run.py”, line 427, in rerun_fws

lp.rerun_fw(int(f), clear_recovery=args.clear_recovery)

File “/b/home/ipgs/dmichea/venv_a2s/lib/python3.6/site-packages/fireworks/core/launchpad.py”, line 1260, in rerun_fw

updated_ids = wf.rerun_fw(fw_id)

File “/b/home/ipgs/dmichea/venv_a2s/lib/python3.6/site-packages/fireworks/core/firework.py”, line 863, in rerun_fw

updated_ids = updated_ids.union(self.rerun_fw(child_id, updated_ids))

File “/b/home/ipgs/dmichea/venv_a2s/lib/python3.6/site-packages/fireworks/core/firework.py”, line 863, in rerun_fw

updated_ids = updated_ids.union(self.rerun_fw(child_id, updated_ids))

File “/b/home/ipgs/dmichea/venv_a2s/lib/python3.6/site-packages/fireworks/core/firework.py”, line 863, in rerun_fw

updated_ids = updated_ids.union(self.rerun_fw(child_id, updated_ids))

[Previous line repeated 986 more times]

File “/b/home/ipgs/dmichea/venv_a2s/lib/python3.6/site-packages/fireworks/core/firework.py”, line 858, in rerun_fw

m_fw._rerun()

File “/b/home/ipgs/dmichea/venv_a2s/lib/python3.6/site-packages/fireworks/core/launchpad.py”, line 1592, in _rerun

self.full_fw._rerun()

File “/b/home/ipgs/dmichea/venv_a2s/lib/python3.6/site-packages/fireworks/core/launchpad.py”, line 1692, in full_fw

self._get_launch_data(launch_field)

File “/b/home/ipgs/dmichea/venv_a2s/lib/python3.6/site-packages/fireworks/core/launchpad.py”, line 1707, in _get_launch_data

fw = self.partial_fw # assure stage 1

RecursionError: maximum recursion depth exceeded

$ lpad get_fws -s FIZZLED

[

{

“fw_id”: 165,

“created_on”: “2017-06-30T08:59:50.860965”,

“updated_on”: “2017-06-30T09:00:51.518826”,

“state”: “FIZZLED”,

“name”: “Pre-correlation: 8937 vs 8931”

},

{

“fw_id”: 166,

“created_on”: “2017-06-30T08:59:50.835693”,

“updated_on”: “2017-06-30T09:00:51.520577”,

“state”: “FIZZLED”,

“name”: “Pre-correlation: 8942 vs 8931”

},

{

“fw_id”: 171,

“created_on”: “2017-06-30T08:59:50.834089”,

“updated_on”: “2017-06-30T09:00:51.835825”,

“state”: “FIZZLED”,

“name”: “Pre-correlation: 8964 vs 8931”

},

{

“fw_id”: 177,

“created_on”: “2017-06-30T08:59:50.843329”,

“updated_on”: “2017-06-30T09:00:51.508829”,

“state”: “FIZZLED”,

“name”: “Pre-correlation: 8970 vs 8931”

},

{

“fw_id”: 178,

“created_on”: “2017-06-30T08:59:50.831047”,

“updated_on”: “2017-06-30T09:00:51.522512”,

“state”: “FIZZLED”,

“name”: “Pre-correlation: 8964 vs 8937”

},

{

“fw_id”: 181,

“created_on”: “2017-06-30T08:59:50.841427”,

“updated_on”: “2017-06-30T09:00:51.825738”,

“state”: “FIZZLED”,

“name”: “Pre-correlation: 8970 vs 8937”

}

]

$ lpad rerun_fws -i 181

Traceback (most recent call last):

File “/b/home/ipgs/dmichea/venv_a2s/bin/lpad”, line 11, in

load_entry_point(‘FireWorks==1.4.1’, ‘console_scripts’, ‘lpad’)()

File “/b/home/ipgs/dmichea/venv_a2s/lib/python3.6/site-packages/fireworks/scripts/lpad_run.py”, line 1053, in lpad

args.func(args)

File “/b/home/ipgs/dmichea/venv_a2s/lib/python3.6/site-packages/fireworks/scripts/lpad_run.py”, line 427, in rerun_fws

lp.rerun_fw(int(f), clear_recovery=args.clear_recovery)

File “/b/home/ipgs/dmichea/venv_a2s/lib/python3.6/site-packages/fireworks/core/launchpad.py”, line 1260, in rerun_fw

updated_ids = wf.rerun_fw(fw_id)

File “/b/home/ipgs/dmichea/venv_a2s/lib/python3.6/site-packages/fireworks/core/firework.py”, line 863, in rerun_fw

updated_ids = updated_ids.union(self.rerun_fw(child_id, updated_ids))

File “/b/home/ipgs/dmichea/venv_a2s/lib/python3.6/site-packages/fireworks/core/firework.py”, line 863, in rerun_fw

updated_ids = updated_ids.union(self.rerun_fw(child_id, updated_ids))

File “/b/home/ipgs/dmichea/venv_a2s/lib/python3.6/site-packages/fireworks/core/firework.py”, line 863, in rerun_fw

updated_ids = updated_ids.union(self.rerun_fw(child_id, updated_ids))

[Previous line repeated 986 more times]

File “/b/home/ipgs/dmichea/venv_a2s/lib/python3.6/site-packages/fireworks/core/firework.py”, line 858, in rerun_fw

m_fw._rerun()

File “/b/home/ipgs/dmichea/venv_a2s/lib/python3.6/site-packages/fireworks/core/launchpad.py”, line 1592, in _rerun

self.full_fw._rerun()

File “/b/home/ipgs/dmichea/venv_a2s/lib/python3.6/site-packages/fireworks/core/launchpad.py”, line 1692, in full_fw

self._get_launch_data(launch_field)

File “/b/home/ipgs/dmichea/venv_a2s/lib/python3.6/site-packages/fireworks/core/launchpad.py”, line 1707, in _get_launch_data

fw = self.partial_fw # assure stage 1

RecursionError: maximum recursion depth exceeded

Hi David

The _fw_name for the custom Firetask would not be the issue. The intended meaning of the docs is that this name should be distinct from all the other Firetask names (e.g. “ScriptTask”). It is expected that whatever name you choose (e.g., “AppendAttributesTask”) - that name will stay constant, just like you have written.

There are two possible reasons I can think of based on the information provided for the error:

  1. The workflow has too many Fireworks. This seems unlikely, but because I haven’t tested extremely large workflows it’s certainly possible. Can you tell me how many different Fireworks are in your workflow that won’t rerun? is it 10, 100, 1000, 10000, 100000, etc? So far, I do not know of a limit, but perhaps you’ve found one…

  2. If it isn’t (1), then it’s possible that somehow your custom Firetask is adding the workflow using FWAction in a way that is creating a cycle in your workflow graph (or, unintentionally linking to fws in other workflows). You could check this by inspecting the parent_links and child_links keys in your “workflows” collection in MongoDB to try to find cycles. Search for {“nodes”: 1} where “1” is your fw_id that won’t rerun and inspect that document. In general, FireWorks is designed so that the FWAction would not create such cycles or unexpected links, but if you are doing something unexpected in the line
    attributes_wf = attributes_S2.get_attributes_wf(src_i)

such as manually linking to fw_ids that already exist somewhere else in your database, that could cause problems.

My inkling is that it’s #2.

Best,

Anubhav

···

On Friday, June 30, 2017 at 6:39:45 AM UTC-7, [email protected] wrote:

Well, I just figured out som I use custom firetask created to build and append new worflow to the current:

class AppendAttributesTask(FireTaskBase):

_fw_name = ‘AppendAttributesTask’

required_params = [“src_id”]

def run_task(self, fw_spec):

  (A2S_logger, __) = A2S_config.init_loggers()
  src_id = self["src_id"]
  attributes_wf = attributes_S2.get_attributes_wf(src_id)
  if attributes_wf:
  	A2S_logger.info("create & append attributes computation workflow")
  	return FWAction(additions=attributes_wf)
  else:
  	A2S_logger.fatal("attributes_S2.get_attributes_wf failed : no workflow created for src id %s" % str(src_id))
  	return FWAction()

``

As you can see, I defined a fixed _fw_name for this task. I read in the doc that it have to be unique. I use this task only once in each workflow, but I run several workflows at the same time which use this task.

I suppose that in this case, the name is not unique in the Mongo DB and that the problem from my previous post should come from this fact …

It seems me quite weird that I can not have several instances of a FireTask class in my workflows …

How can I overcome this problem ? Should I define a dynamic name to avoid name collisions (with a randomly generated suffix for example) ?

Best regards,
David

Le vendredi 30 juin 2017 11:15:17 UTC+2, [email protected] a écrit :

Hi,

I am currently debugging a workflow, but when I try to rerun FIZZLED FWs, it raises errors : “RecursionError: maximum recursion depth exceeded”

Note that these FWs are in a workflow dynamically generated and append by a custom firetask.

Is there a way to rerun these fireworks without deleting and rebuilding the entire workflow ?

How can I avoid such problems ?

Best regards,
David

Here are some tracebacks :

$ lpad rerun_fws -s FIZZLED

Traceback (most recent call last):

File “/b/home/ipgs/dmichea/venv_a2s/bin/lpad”, line 11, in

load_entry_point(‘FireWorks==1.4.1’, ‘console_scripts’, ‘lpad’)()

File “/b/home/ipgs/dmichea/venv_a2s/lib/python3.6/site-packages/fireworks/scripts/lpad_run.py”, line 1053, in lpad

args.func(args)

File “/b/home/ipgs/dmichea/venv_a2s/lib/python3.6/site-packages/fireworks/scripts/lpad_run.py”, line 427, in rerun_fws

lp.rerun_fw(int(f), clear_recovery=args.clear_recovery)

File “/b/home/ipgs/dmichea/venv_a2s/lib/python3.6/site-packages/fireworks/core/launchpad.py”, line 1260, in rerun_fw

updated_ids = wf.rerun_fw(fw_id)

File “/b/home/ipgs/dmichea/venv_a2s/lib/python3.6/site-packages/fireworks/core/firework.py”, line 863, in rerun_fw

updated_ids = updated_ids.union(self.rerun_fw(child_id, updated_ids))

File “/b/home/ipgs/dmichea/venv_a2s/lib/python3.6/site-packages/fireworks/core/firework.py”, line 863, in rerun_fw

updated_ids = updated_ids.union(self.rerun_fw(child_id, updated_ids))

File “/b/home/ipgs/dmichea/venv_a2s/lib/python3.6/site-packages/fireworks/core/firework.py”, line 863, in rerun_fw

updated_ids = updated_ids.union(self.rerun_fw(child_id, updated_ids))

[Previous line repeated 986 more times]

File “/b/home/ipgs/dmichea/venv_a2s/lib/python3.6/site-packages/fireworks/core/firework.py”, line 858, in rerun_fw

m_fw._rerun()

File “/b/home/ipgs/dmichea/venv_a2s/lib/python3.6/site-packages/fireworks/core/launchpad.py”, line 1592, in _rerun

self.full_fw._rerun()

File “/b/home/ipgs/dmichea/venv_a2s/lib/python3.6/site-packages/fireworks/core/launchpad.py”, line 1692, in full_fw

self._get_launch_data(launch_field)

File “/b/home/ipgs/dmichea/venv_a2s/lib/python3.6/site-packages/fireworks/core/launchpad.py”, line 1707, in _get_launch_data

fw = self.partial_fw # assure stage 1

RecursionError: maximum recursion depth exceeded

$ lpad get_fws -s FIZZLED

[

{

“fw_id”: 165,

“created_on”: “2017-06-30T08:59:50.860965”,

“updated_on”: “2017-06-30T09:00:51.518826”,

“state”: “FIZZLED”,

“name”: “Pre-correlation: 8937 vs 8931”

},

{

“fw_id”: 166,

“created_on”: “2017-06-30T08:59:50.835693”,

“updated_on”: “2017-06-30T09:00:51.520577”,

“state”: “FIZZLED”,

“name”: “Pre-correlation: 8942 vs 8931”

},

{

“fw_id”: 171,

“created_on”: “2017-06-30T08:59:50.834089”,

“updated_on”: “2017-06-30T09:00:51.835825”,

“state”: “FIZZLED”,

“name”: “Pre-correlation: 8964 vs 8931”

},

{

“fw_id”: 177,

“created_on”: “2017-06-30T08:59:50.843329”,

“updated_on”: “2017-06-30T09:00:51.508829”,

“state”: “FIZZLED”,

“name”: “Pre-correlation: 8970 vs 8931”

},

{

“fw_id”: 178,

“created_on”: “2017-06-30T08:59:50.831047”,

“updated_on”: “2017-06-30T09:00:51.522512”,

“state”: “FIZZLED”,

“name”: “Pre-correlation: 8964 vs 8937”

},

{

“fw_id”: 181,

“created_on”: “2017-06-30T08:59:50.841427”,

“updated_on”: “2017-06-30T09:00:51.825738”,

“state”: “FIZZLED”,

“name”: “Pre-correlation: 8970 vs 8937”

}

]

$ lpad rerun_fws -i 181

Traceback (most recent call last):

File “/b/home/ipgs/dmichea/venv_a2s/bin/lpad”, line 11, in

load_entry_point(‘FireWorks==1.4.1’, ‘console_scripts’, ‘lpad’)()

File “/b/home/ipgs/dmichea/venv_a2s/lib/python3.6/site-packages/fireworks/scripts/lpad_run.py”, line 1053, in lpad

args.func(args)

File “/b/home/ipgs/dmichea/venv_a2s/lib/python3.6/site-packages/fireworks/scripts/lpad_run.py”, line 427, in rerun_fws

lp.rerun_fw(int(f), clear_recovery=args.clear_recovery)

File “/b/home/ipgs/dmichea/venv_a2s/lib/python3.6/site-packages/fireworks/core/launchpad.py”, line 1260, in rerun_fw

updated_ids = wf.rerun_fw(fw_id)

File “/b/home/ipgs/dmichea/venv_a2s/lib/python3.6/site-packages/fireworks/core/firework.py”, line 863, in rerun_fw

updated_ids = updated_ids.union(self.rerun_fw(child_id, updated_ids))

File “/b/home/ipgs/dmichea/venv_a2s/lib/python3.6/site-packages/fireworks/core/firework.py”, line 863, in rerun_fw

updated_ids = updated_ids.union(self.rerun_fw(child_id, updated_ids))

File “/b/home/ipgs/dmichea/venv_a2s/lib/python3.6/site-packages/fireworks/core/firework.py”, line 863, in rerun_fw

updated_ids = updated_ids.union(self.rerun_fw(child_id, updated_ids))

[Previous line repeated 986 more times]

File “/b/home/ipgs/dmichea/venv_a2s/lib/python3.6/site-packages/fireworks/core/firework.py”, line 858, in rerun_fw

m_fw._rerun()

File “/b/home/ipgs/dmichea/venv_a2s/lib/python3.6/site-packages/fireworks/core/launchpad.py”, line 1592, in _rerun

self.full_fw._rerun()

File “/b/home/ipgs/dmichea/venv_a2s/lib/python3.6/site-packages/fireworks/core/launchpad.py”, line 1692, in full_fw

self._get_launch_data(launch_field)

File “/b/home/ipgs/dmichea/venv_a2s/lib/python3.6/site-packages/fireworks/core/launchpad.py”, line 1707, in _get_launch_data

fw = self.partial_fw # assure stage 1

RecursionError: maximum recursion depth exceeded

Hi Anubhav,

I finally ended-up to find the problem : on a worker, I had let an old version of my own firetasks. They was defined within a .py file for each of them. After that, I have put all of them in a single file, but I had forgot to apply these change on a single worker in charge of the creation of some workflows. This has lead to name conflict in the MongoDB …

I have corrected this and finally all is working correctly.

Thanks a lot for your quick answers,
David

···

Le vendredi 30 juin 2017 18:00:14 UTC+2, Anubhav Jain a écrit :

Hi David

The _fw_name for the custom Firetask would not be the issue. The intended meaning of the docs is that this name should be distinct from all the other Firetask names (e.g. “ScriptTask”). It is expected that whatever name you choose (e.g., “AppendAttributesTask”) - that name will stay constant, just like you have written.

There are two possible reasons I can think of based on the information provided for the error:

  1. The workflow has too many Fireworks. This seems unlikely, but because I haven’t tested extremely large workflows it’s certainly possible. Can you tell me how many different Fireworks are in your workflow that won’t rerun? is it 10, 100, 1000, 10000, 100000, etc? So far, I do not know of a limit, but perhaps you’ve found one…
  1. If it isn’t (1), then it’s possible that somehow your custom Firetask is adding the workflow using FWAction in a way that is creating a cycle in your workflow graph (or, unintentionally linking to fws in other workflows). You could check this by inspecting the parent_links and child_links keys in your “workflows” collection in MongoDB to try to find cycles. Search for {“nodes”: 1} where “1” is your fw_id that won’t rerun and inspect that document. In general, FireWorks is designed so that the FWAction would not create such cycles or unexpected links, but if you are doing something unexpected in the line
    attributes_wf = attributes_S2.get_attributes_wf(src_i)

such as manually linking to fw_ids that already exist somewhere else in your database, that could cause problems.

My inkling is that it’s #2.

Best,

Anubhav

On Friday, June 30, 2017 at 6:39:45 AM UTC-7, [email protected] wrote:

Well, I just figured out som I use custom firetask created to build and append new worflow to the current:

class AppendAttributesTask(FireTaskBase):

_fw_name = 'AppendAttributesTask'
required_params = ["src_id"]
def run_task(self, fw_spec):
  (A2S_logger, __) = A2S_config.init_loggers()
  src_id = self["src_id"]
  attributes_wf = attributes_S2.get_attributes_wf(src_id)
  if attributes_wf:
  	A2S_logger.info("create & append attributes computation workflow")
  	return FWAction(additions=attributes_wf)
  else:
  	A2S_logger.fatal("attributes_S2.get_attributes_wf failed : no workflow created for src id %s" % str(src_id))
  	return FWAction()

``

As you can see, I defined a fixed _fw_name for this task. I read in the doc that it have to be unique. I use this task only once in each workflow, but I run several workflows at the same time which use this task.

I suppose that in this case, the name is not unique in the Mongo DB and that the problem from my previous post should come from this fact …

It seems me quite weird that I can not have several instances of a FireTask class in my workflows …

How can I overcome this problem ? Should I define a dynamic name to avoid name collisions (with a randomly generated suffix for example) ?

Best regards,
David

Le vendredi 30 juin 2017 11:15:17 UTC+2, [email protected] a écrit :

Hi,

I am currently debugging a workflow, but when I try to rerun FIZZLED FWs, it raises errors : “RecursionError: maximum recursion depth exceeded”

Note that these FWs are in a workflow dynamically generated and append by a custom firetask.

Is there a way to rerun these fireworks without deleting and rebuilding the entire workflow ?

How can I avoid such problems ?

Best regards,
David

Here are some tracebacks :

$ lpad rerun_fws -s FIZZLED

Traceback (most recent call last):

File “/b/home/ipgs/dmichea/venv_a2s/bin/lpad”, line 11, in

load_entry_point(‘FireWorks==1.4.1’, ‘console_scripts’, ‘lpad’)()

File “/b/home/ipgs/dmichea/venv_a2s/lib/python3.6/site-packages/fireworks/scripts/lpad_run.py”, line 1053, in lpad

args.func(args)

File “/b/home/ipgs/dmichea/venv_a2s/lib/python3.6/site-packages/fireworks/scripts/lpad_run.py”, line 427, in rerun_fws

lp.rerun_fw(int(f), clear_recovery=args.clear_recovery)

File “/b/home/ipgs/dmichea/venv_a2s/lib/python3.6/site-packages/fireworks/core/launchpad.py”, line 1260, in rerun_fw

updated_ids = wf.rerun_fw(fw_id)

File “/b/home/ipgs/dmichea/venv_a2s/lib/python3.6/site-packages/fireworks/core/firework.py”, line 863, in rerun_fw

updated_ids = updated_ids.union(self.rerun_fw(child_id, updated_ids))

File “/b/home/ipgs/dmichea/venv_a2s/lib/python3.6/site-packages/fireworks/core/firework.py”, line 863, in rerun_fw

updated_ids = updated_ids.union(self.rerun_fw(child_id, updated_ids))

File “/b/home/ipgs/dmichea/venv_a2s/lib/python3.6/site-packages/fireworks/core/firework.py”, line 863, in rerun_fw

updated_ids = updated_ids.union(self.rerun_fw(child_id, updated_ids))

[Previous line repeated 986 more times]

File “/b/home/ipgs/dmichea/venv_a2s/lib/python3.6/site-packages/fireworks/core/firework.py”, line 858, in rerun_fw

m_fw._rerun()

File “/b/home/ipgs/dmichea/venv_a2s/lib/python3.6/site-packages/fireworks/core/launchpad.py”, line 1592, in _rerun

self.full_fw._rerun()

File “/b/home/ipgs/dmichea/venv_a2s/lib/python3.6/site-packages/fireworks/core/launchpad.py”, line 1692, in full_fw

self._get_launch_data(launch_field)

File “/b/home/ipgs/dmichea/venv_a2s/lib/python3.6/site-packages/fireworks/core/launchpad.py”, line 1707, in _get_launch_data

fw = self.partial_fw # assure stage 1

RecursionError: maximum recursion depth exceeded

$ lpad get_fws -s FIZZLED

[

{

“fw_id”: 165,

“created_on”: “2017-06-30T08:59:50.860965”,

“updated_on”: “2017-06-30T09:00:51.518826”,

“state”: “FIZZLED”,

“name”: “Pre-correlation: 8937 vs 8931”

},

{

“fw_id”: 166,

“created_on”: “2017-06-30T08:59:50.835693”,

“updated_on”: “2017-06-30T09:00:51.520577”,

“state”: “FIZZLED”,

“name”: “Pre-correlation: 8942 vs 8931”

},

{

“fw_id”: 171,

“created_on”: “2017-06-30T08:59:50.834089”,

“updated_on”: “2017-06-30T09:00:51.835825”,

“state”: “FIZZLED”,

“name”: “Pre-correlation: 8964 vs 8931”

},

{

“fw_id”: 177,

“created_on”: “2017-06-30T08:59:50.843329”,

“updated_on”: “2017-06-30T09:00:51.508829”,

“state”: “FIZZLED”,

“name”: “Pre-correlation: 8970 vs 8931”

},

{

“fw_id”: 178,

“created_on”: “2017-06-30T08:59:50.831047”,

“updated_on”: “2017-06-30T09:00:51.522512”,

“state”: “FIZZLED”,

“name”: “Pre-correlation: 8964 vs 8937”

},

{

“fw_id”: 181,

“created_on”: “2017-06-30T08:59:50.841427”,

“updated_on”: “2017-06-30T09:00:51.825738”,

“state”: “FIZZLED”,

“name”: “Pre-correlation: 8970 vs 8937”

}

]

$ lpad rerun_fws -i 181

Traceback (most recent call last):

File “/b/home/ipgs/dmichea/venv_a2s/bin/lpad”, line 11, in

load_entry_point(‘FireWorks==1.4.1’, ‘console_scripts’, ‘lpad’)()

File “/b/home/ipgs/dmichea/venv_a2s/lib/python3.6/site-packages/fireworks/scripts/lpad_run.py”, line 1053, in lpad

args.func(args)

File “/b/home/ipgs/dmichea/venv_a2s/lib/python3.6/site-packages/fireworks/scripts/lpad_run.py”, line 427, in rerun_fws

lp.rerun_fw(int(f), clear_recovery=args.clear_recovery)

File “/b/home/ipgs/dmichea/venv_a2s/lib/python3.6/site-packages/fireworks/core/launchpad.py”, line 1260, in rerun_fw

updated_ids = wf.rerun_fw(fw_id)

File “/b/home/ipgs/dmichea/venv_a2s/lib/python3.6/site-packages/fireworks/core/firework.py”, line 863, in rerun_fw

updated_ids = updated_ids.union(self.rerun_fw(child_id, updated_ids))

File “/b/home/ipgs/dmichea/venv_a2s/lib/python3.6/site-packages/fireworks/core/firework.py”, line 863, in rerun_fw

updated_ids = updated_ids.union(self.rerun_fw(child_id, updated_ids))

File “/b/home/ipgs/dmichea/venv_a2s/lib/python3.6/site-packages/fireworks/core/firework.py”, line 863, in rerun_fw

updated_ids = updated_ids.union(self.rerun_fw(child_id, updated_ids))

[Previous line repeated 986 more times]

File “/b/home/ipgs/dmichea/venv_a2s/lib/python3.6/site-packages/fireworks/core/firework.py”, line 858, in rerun_fw

m_fw._rerun()

File “/b/home/ipgs/dmichea/venv_a2s/lib/python3.6/site-packages/fireworks/core/launchpad.py”, line 1592, in _rerun

self.full_fw._rerun()

File “/b/home/ipgs/dmichea/venv_a2s/lib/python3.6/site-packages/fireworks/core/launchpad.py”, line 1692, in full_fw

self._get_launch_data(launch_field)

File “/b/home/ipgs/dmichea/venv_a2s/lib/python3.6/site-packages/fireworks/core/launchpad.py”, line 1707, in _get_launch_data

fw = self.partial_fw # assure stage 1

RecursionError: maximum recursion depth exceeded